Randomly select layer in tensorflow model

user_16166807 0
Viewed 187

I want to use different layers with specific probabilities in my network. Layers are the following classes.

class plus1(keras.layers.Layer):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
    def call(self, X):
        return X + 1    
    def compute_output_shape(self, batch_input_shape):
        return batch_input_shape

class plus2(keras.layers.Layer):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
    def call(self, X):
        return X + 2
    def compute_output_shape(self, batch_input_shape):
        return batch_input_shape

class plus3(keras.layers.Layer):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
    def call(self, X):
        return X + 3
    def compute_output_shape(self, batch_input_shape):
        return batch_input_shape

And the network is like below.

def f1():
    return plus1()
def f2():
    return plus2()
def f3():
    return plus3()

def simple_model(input_num):
    input_layer = Input(input_num)
    rand = tf.random.uniform((1,), minval=0, maxval=3, dtype=tf.int32)
    r = tf.switch_case(rand[0], branch_fns={0: f1, 1: f2, 2: f3})
    res = r(input_layer)
    model = Model(inputs=input_layer, outputs=res)
    return model

model = simple_model([1,])

Each time I run the code below, I get the same output, but I expected different ones. Is there any way to implement this?

model.predict([1])
>>> array([[4.]], dtype=float32)
keras tensorflow layer python
2 Answers

It was the same problem I was facing and I found no solution. So I implemented distinct networks and then randomly selected from their output.

user_110530070

I've been dealing with the same issue: I have a list of layers from which I need to select randomly on every iteration. tf.switch_case() gave me the same problem you describe.

For whatever reason, and I don't have enough background depth to tell you why (it's entirely plausible that my tf.switch_case implementation was buggy in an unrelated way), this code worked for me:

def random_layer(layers, image_tensor):
"""
Selects and executes a random layer chosen from a list
"""
to_use = tf.random.uniform(shape=[], maxval=len(layers), dtype=tf.int32)
out = image_tensor

for i, layer in enumerate(layers):
    # out is either image_tensor or the actual output, *but*
    # since we can't break this loop, when it matches it will become the actual output
    # and any further calls will return that value
    
    def _match():
        # tf.print("using {}".format(layer))
        return layer(out, training=True)
    out = tf.cond(to_use==i, _match, lambda: out)
    
return out

(Note that I'm using a local function just so I can verify the randomness.) And I pass in:

NOISE_LAYERS = [tf.keras.layers.GaussianNoise(stddev=.1),
               tf.keras.layers.GaussianNoise(stddev=.2),
               tf.keras.layers.GaussianNoise(stddev=.3),
               tf.keras.layers.GaussianNoise(stddev=.4)]

(This is part of dataset preparation in which I want images to contain varying amounts of noise.)

user_180676570
Related