Random seed in tensorflow

Viewed 185

I am trying to implement custom attention layer for NLP task in tensorflow:

os.environ['PYTHONHASHSEED']=str(1)
tf.random.set_seed(1)
np.random.seed(1)
random.seed(1) 

from tensorflow.keras.layers import Layer
import tensorflow.keras.backend as K
class attention(Layer):
    def __init__(self,**kwargs):
        super(attention,self).__init__(**kwargs)

def build(self,input_shape):
    self.W=self.add_weight(name="att_weight",shape=(input_shape[-1],1),initializer="normal")
    self.b=self.add_weight(name="att_bias",shape=(input_shape[1],1),initializer="zeros")        
    super(attention, self).build(input_shape)

def call(self,x):
    et=K.squeeze(K.tanh(K.dot(x,self.W)+self.b),axis=-1)
    at=K.softmax(et)
    at=K.expand_dims(at,axis=-1)
    output=x*at
    return K.sum(output,axis=1)

def compute_output_shape(self,input_shape):
    return (input_shape[0],input_shape[-1])

def get_config(self):
    return super(attention,self).get_config()

The only way to implement it that in tensorflow I found is the following:

import random
os.environ['PYTHONHASHSEED']=str(1)
tf.random.set_seed(1)
np.random.seed(1)
random.seed(1) 

vocab_size = len(tokenizer.word_index) + 1
embedding_dim = 16
max_length = 100
trunc_type='post'
padding_type='post'
oov_tok = "<OOV>"

training_sequences = tokenizer.texts_to_sequences(df.Message)
training_padded = pad_sequences(training_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)

valid_sequences = tokenizer.texts_to_sequences(valid.Message)
valid_padded = pad_sequences(valid_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)

y_train = to_categorical(df.Category, 3)
y_test = to_categorical(valid.Category, 3)

X_resampled, y_resampled = SMOTE(random_state=1).fit_resample(training_padded, y_train)

import random
os.environ['PYTHONHASHSEED']=str(1)
tf.random.set_seed(1)
np.random.seed(1)
random.seed(1) 

inputs=Input((features,))
x=Embedding(input_dim=vocab_size+1,output_dim=32,input_length=features,\
            embeddings_regularizer=keras.regularizers.l2(.001))(inputs)
att_in=LSTM(10,return_sequences=True,dropout=0.3,recurrent_dropout=0.2)(x)
att_out=attention()(att_in)
den6=Dense(100,activation='relu',trainable=True)(att_out)
den7=Dense(50,activation='relu',trainable=True)(den6)
den=Dense(25,activation='relu',trainable=True)(den7)
outputs=Dense(3,activation='softmax',trainable=True)(den)
model=Model(inputs,outputs)

model.compile(loss='CategoricalCrossentropy',optimizer='adam',weighted_metrics=['accuracy'])
model.fit(X_resampled, y_resampled, epochs=3, validation_data=(valid_padded, y_test), batch_size = 150, verbose=1)

But results are not reproducible. Even if I put

import random
os.environ['PYTHONHASHSEED']=str(1)
tf.random.set_seed(1)
np.random.seed(1)
random.seed(1) 

before every layer, I get different results every time. Is there a way I can make it work? To sum up: this model works fine but I would like the result to be consistent every time that I run the code so that anyone can reproduce result that I have achieved with my model implementation.

0 Answers
Related