How to put a Max-Min constraint on a hidden Dense Layer?

user_7875842 0
Viewed 623

I am trying to understand how to put a constraint on a dense hidden layer in a model like the following one.

I would like to develop the function applyConstraint which rescale the values contained in params between the values min and max.

How would you do it?

inp = tfl.Input((10,))
dense = tfl.Dense(16, activation = 'relu')(inp)
dense = tfl.Dense(8, activation = 'relu')(dense)
params = tfl.Dense(3, activation = 'relu')(dense)
params_max_min = applyConstraint(params, min, max)
concat = tfl.Concatenate()([dense, params])
dense = tfl.Dense(16, activation = 'relu')(concat)
dense = tfl.Dense(8, activation = 'relu')(dense)
dense = tfl.Dense(1, activation = None)(dense)
model = tf.keras.Model(inputs = inp, outputs = dense)
model_params = tf.keras.Model(inputs = inp, outputs = params_max_min)
model.compile(optimizer = 'adam', loss = 'mse')
keras tensorflow normalization
2 Answers

You can subclass tf.keras.constraints.Constraint and customize your own op to do what you want.

Define constraint:

import numpy as np
import tensorflow as tf

from tensorflow.keras.constraints import Constraint


class MinMaxConstraint(Constraint):
    """constrain model weights between [x_min, x_max]."""
    def __init__(self, x_min=0.0, x_max=1.0):
        super().__init__()
        self.x_min = x_min
        self.x_max = x_max
        # TODO: add sanity check if x_max == x_min or w_max == w_min
    
    def __call__(self, w):
        w_min = tf.minimum(tf.math.reduce_min(w), self.x_min)
        w_max = tf.maximum(tf.math.reduce_max(w), self.x_max)
        scale = (self.x_max - self.x_min) / (w_max - w_min)
        m = self.x_min - w_min * scale
        w = w * scale
        return w + m

Test on default case:

# random data
X = tf.random.normal([10, 2])
y = tf.random.normal([10])

# optimizer
m_opt = tf.keras.optimizers.Adam(1e-3)

# network definition
x_in = tf.keras.Input([2])
x = tf.keras.layers.Dense(4, kernel_constraint=MinMaxConstraint())(x_in)
x_out = tf.keras.layers.Dense(1)(x)

# model definition
model = tf.keras.models.Model(x_in, x_out)

# do a forward pass and update
with tf.GradientTape() as tape:
    y_hat = model(X)
    loss = tf.math.reduce_mean(tf.losses.MSE(y, y_hat))
  
m_vars = model.trainable_variables
m_grads = tape.gradient(loss, m_vars)
m_opt.apply_gradients(zip(m_grads, m_vars))

# check weights
assert np.all(model.get_weights()[0] >= 0.0)
assert np.all(model.get_weights()[0] <= 1.0)
# passes!

Test on [-2, 2]:

# reset network
x_in = tf.keras.Input([2])
x = tf.keras.layers.Dense(4, kernel_constraint=MinMaxConstraint(-2.0, 2.0))(x_in)
x_out = tf.keras.layers.Dense(1)(x)

# reset model
model = tf.keras.models.Model(x_in, x_out)

# do a forward pass and update
with tf.GradientTape() as tape:
    y_hat = model(X)
    loss = tf.math.reduce_mean(tf.losses.MSE(y, y_hat))
  
m_vars = model.trainable_variables
m_grads = tape.gradient(loss, m_vars)
m_opt.apply_gradients(zip(m_grads, m_vars))

# check weights again
assert np.all(model.get_weights()[0] >= -2.0)
assert np.all(model.get_weights()[0] <= 2.0)
# passes!
user_32227970

Try this:

import tensorflow as tf
params = tf.random.uniform((2, 3))
min, max = 4., 5.
def applyConstraint(params, min, max):
  mn = tf.reduce_min(params)
  mx = tf.reduce_max(params)
  mult = (max-min)/(mx-mn)
  p = min + (params - mn) * mult
  return p

output = applyConstraint(params, min, max)
user_55614720
Related