I try to write a seq2seq decoder with the tensorflow tf.contrib.seq2seq package. I am wondering if my code is correct and if there is better way to rewrite it. The documentation is not easy to read.
Or my question can be: how can I easily debug this kind of code? How can I inspect some intermediate results in tensorflow?
class Decoder:
def __init__(self, embedding, hidden_size, num_layers=1, max_length=15):
self.embedding = embedding
self.hidden_size = hidden_size
self.num_layers = num_layers
self.cell = tf.nn.rnn_cell.GRUCell(hidden_size)
self.linear = tf.Variable(tf.random_normal(shape=(self.hidden_size, cn_total_words))*0.1)
def __call__(self, inputs, state, encoder_outputs, encoder_state, decoder_length, mode="train"):
with tf.variable_scope("decoder") as scope:
inputs = tf.nn.embedding_lookup(self.embedding, inputs)
encoder_state = tf.tile(tf.expand_dims(encoder_state, 1), (1, tf.shape(inputs)[1], 1))
attention_mechanism = tf.contrib.seq2seq.LuongAttention(self.hidden_size, encoder_outputs)
attn_cell = tf.contrib.seq2seq.AttentionWrapper(self.cell, attention_mechanism, self.hidden_size)
if mode == "train":
helper = tf.contrib.seq2seq.TrainingHelper(inputs=inputs, sequence_length=decoder_length)
elif mode == "infer":
helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(embedding=self.embedding,
start_tokens=tf.tile([en_dict["BOS"]], [tf.shape(inputs)[0]]), end_token=en_dict["EOS"])
decoder = tf.contrib.seq2seq.BasicDecoder(cell=attn_cell, helper=helper,
initial_state=attn_cell.zero_state(tf.shape(inputs)[0], tf.float32))
outputs, _, _ = tf.contrib.seq2seq.dynamic_decode(decoder=decoder)
outputs = tf.concat([tf.expand_dims(out, 1) for out in outputs], 1)
outputs = tf.tensordot(outputs, self.linear, axes=[[2], [0]])
return outputs, state
I got the following error when running the code
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) ~/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py in apply_op(self, op_type_name, name, **keywords) 434 preferred_dtype=default_dtype, --> 435 as_ref=input_arg.is_ref) 436 if input_arg.number_attr and len(
~/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in internal_convert_n_to_tensor(values, dtype, name, as_ref, preferred_dtype) 736 as_ref=as_ref, --> 737 preferred_dtype=preferred_dtype)) 738 return ret
~/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in internal_convert_to_tensor(value, dtype, name, as_ref, preferred_dtype) 675 if ret is None: --> 676 ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref) 677
~/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in _TensorTensorConversionFunction(t, dtype, name, as_ref) 548 "Tensor conversion requested dtype %s for Tensor with dtype %s: %r" --> 549 % (dtype.name, t.dtype.name, str(t))) 550 return t
ValueError: Tensor conversion requested dtype float32 for Tensor with dtype int32: 'Tensor("seq2seq-train/decoder/ExpandDims_2:0", shape=(?, 1, ?), dtype=int32)'
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last) in () 4 emb_en = np.random.uniform(low=-0.1, high=0.1, size=(en_total_words, hidden_size)) 5 emb_cn = np.random.uniform(low=-0.1, high=0.1, size=(cn_total_words, hidden_size)) ----> 6 model = Seq2Seq(hidden_size, num_layers, emb_en, emb_cn) 7 sess = tf.Session() 8 init = tf.global_variables_initializer()
in init(self, hidden_size, num_layers, embed_words_en, embed_words_cn) 81 encoder_outputs, encoder_state = self.encoder(self.encoder_inputs, self.encoder_length) 82 decoder_length = tf.cast(tf.reduce_sum(self.decoder_mask, 1), tf.int32) ---> 83 decoder_outputs, decoder_state = self.decoder(self.decoder_inputs, encoder_state, encoder_outputs, encoder_state, decoder_length) 84 85 # decoder_outputs.append(decoder_out)
in call(self, inputs, state, encoder_outputs, encoder_state, decoder_length, mode) 50 51 outputs, _, _ = tf.contrib.seq2seq.dynamic_decode(decoder=decoder) ---> 52 outputs = tf.concat([tf.expand_dims(out, 1) for out in outputs], 1) 53 54 outputs = tf.tensordot(outputs, self.linear, axes=[[2], [0]])
~/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py in concat(values, axis, name) 1064 return gen_array_ops._concat_v2(values=values, 1065
axis=axis, -> 1066 name=name) 1067 1068~/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py in _concat_v2(values, axis, name) 491 """ 492 result = _op_def_lib.apply_op("ConcatV2", values=values, axis=axis, --> 493 name=name) 494 return result 495
~/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py in apply_op(self, op_type_name, name, **keywords) 461 (prefix, dtype.name)) 462 else: --> 463 raise TypeError("%s that don't all match." % prefix) 464 else: 465 raise TypeError("%s that are invalid." % prefix)
TypeError: Tensors in list passed to 'values' of 'ConcatV2' Op have types [float32, int32] that don't all match.