Tensorflow won't share variable within same graph

user_10905503 0
Viewed 320

I'm testing the tf.variable_scope object to reuse a network multiple times within same session.run call:

data1 = tf.constant([[[3.,5.,6.,1.]]],dtype=tf.float64)
data2 = tf.constant(np.zeros((1,5)))

def networkS(input_1, input_2, reuse):
    #this is a multi-input network using tf.keras api

    with tf.variable_scope("test", reuse=reuse):
        #input_1
        x = tf.keras.layers.CuDNNGRU(512, return_sequences=True)(input_1)
        x = tf.keras.layers.CuDNNGRU(512, return_sequences=True)(x)
        x = tf.keras.layers.CuDNNGRU(512)(x)
        #input_2
        y = tf.keras.layers.Dense(32, activation="relu")(input_2)
        #merge two input
        x = tf.keras.layers.concatenate([x, y], axis=-1)

        x = tf.keras.layers.Dense(512, activation='relu')(x)
        x = tf.keras.layers.Dense(256, activation='relu')(x)
        x = tf.keras.layers.Dense(128, activation='relu')(x)

        vf_pre = tf.keras.layers.Dense(128, activation='relu')(x)
        vf = tf.keras.layers.Dense(1)(vf_pre)
    return vf

v1 = networkS(data1, data2, tf.AUTO_REUSE)
v2 = networkS(data1, data2, tf.AUTO_REUSE)
v3 = networkS(data1, data2, tf.AUTO_REUSE)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    print(sess.run([v1,v2,v3]))

My understanding is that, during the graph construction phase: the first call to networkS(), we create a new network under variavle_scope "test", then the following calls to networkS() just reuse the existing layer variable. However, with identical inputs, we have different results for v1, v2 ,v3

[array([[0.00112361]]), array([[0.00107469]]), array([[0.00115032]])]

I think this means the three network are constructed in paralell and fail to share the same variables, thus producing different results from same inputs.

If I call sess.run twice, it does produce same results between calls

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    print(sess.run([v1,v2,v3]))
    print(sess.run([v1,v2,v3]))

[array([[0.00550815]]), array([[-0.00294633]]), array([[0.00584344]])]
[array([[0.00550815]]), array([[-0.00294633]]), array([[0.00584344]])]

How do I solve this problem?

keras tensorflow python-3.x
1 Answers

Found the solution: According to this keras github post:, it is intended that every time you create a keras.layer class, a new set of variables are created. To reuse variable, you need to create an layer or model object and call the object to reuse variables.

New code:

class GruModel(tf.keras.Model):
def __init__(self):
    super(GruModel, self).__init__()
    #create all the keras.layers objects in __init__()
    self.GRU_1 = tf.keras.layers.CuDNNGRU(512, return_sequences=True)
    self.GRU_2 = tf.keras.layers.CuDNNGRU(512, return_sequences=True)
    self.GRU_end = tf.keras.layers.CuDNNGRU(512)

    self.Dense_second_input = tf.keras.layers.Dense(32, activation="relu")

    self.Dense_merge = tf.keras.layers.Concatenate(axis=-1)

    self.Dense_1 = tf.keras.layers.Dense(512, activation='relu')
    self.Dense_2 = tf.keras.layers.Dense(256, activation='relu')
    self.Dense_3 = tf.keras.layers.Dense(128, activation='relu')

    self.Dense_vf_pre = tf.keras.layers.Dense(128, activation='relu')
    self.Dense_vf = tf.keras.layers.Dense(1)

def call(self, input_1, input_2):
    #input_1
    x = self.GRU_1(input_1)
    x = self.GRU_2(x)
    x = self.GRU_end(x)
    #input_2
    y = self.Dense_second_input(input_2)
    #merge two input
    x = self.Dense_merge.apply([x, y])

    x = self.Dense_1(x)
    x = self.Dense_2(x)
    x = self.Dense_3(x)

    vf_pre = self.Dense_vf_pre(x)
    vf = self.Dense_vf(vf_pre)
    return vf

Then:

data1 = tf.constant([[[3.,5.,6.,1.]]],dtype=tf.float64)
data2 = tf.constant(np.zeros((1,5)))

model = GruModel()

v1 = model.call(data1, data2)
v2 = model.call(data1, data2)
v3 = model.call(data1, data2)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    print(sess.run([v1,v2,v3]))
    print(sess.run([v1,v2,v3]))

Results:

[array([[0.01640865]]), array([[0.01640865]]), array([[0.01640865]])]
[array([[0.01640865]]), array([[0.01640865]]), array([[0.01640865]])]
user_109055030
Related