I am trying to implement multidimensional sequence to sequence Semi Supervised Learning model using Tensorflow's Neural Structured Learning(NSL) Package. My model uses GRU layers where input contains three different features and output contains two different features. So,in this case input shape is float (11,3) and output shape is int (11,2).
I have implemented my NSL model following this tutorial. My understading about tfrecords, nsl data structure and graph regularization isn't enough. Thus, I have been struggling to build correct data preparing. So, I am asking here help to build correct data structure.
I think the first part works fine. I was able to make embeddings, graphs and nls training data without problems. I made artifical data for this purpose and I splited the data into shape (11,5) where the first 3 features are input data and rest of the data are integer vector labels.
import numpy as np
import math
import neural_structured_learning as nsl
import tensorflow as tf
import tensorflow_hub as hub
# Resets notebook state
tf.keras.backend.clear_session()
pretrained_embedding = 'https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1'
hub_layer = hub.KerasLayer(
pretrained_embedding, input_shape=[], dtype=tf.string, trainable=True)
def generate_train_dataset():
n = np.linspace(0,100,1000)
data = np.zeros([1000,5])
data[:,0] = np.sin(n) +np.random.randn(1000)
data[:,1] = np.cos(n) +np.random.randn(1000)
data[:,2] = np.cosh(n) + np.random.randn(1000)
for i in range(100):
data[i:i+9, 3] = 1
data[i-5:i + 4, 4] = 1
return data
data = generate_train_dataset()
def _int64_feature(value):
"""Returns int64 tf.train.Feature."""
return tf.train.Feature(int64_list=tf.train.Int64List(value=value.tolist()))
def _bytes_feature(value):
"""Returns bytes tf.train.Feature."""
return tf.train.Feature(
bytes_list=tf.train.BytesList(value=[value.encode('utf-8')]))
def _float_feature(value):
"""Returns float tf.train.Feature."""
return tf.train.Feature(float_list=tf.train.FloatList(value=value.tolist()))
def create_embedding_example(data_seq, record_id):
"""Create tf.Example containing the sample's embedding and its ID."""
# Shape = [batch_size,].
layer = tf.as_string(tf.reshape(data_seq, shape=[-1, ]),precision=9)
sentence_embedding = hub_layer(layer)
# Flatten the sentence embedding back to 1-D.
sentence_embedding = tf.reshape(sentence_embedding, shape=[-1])
features = {
'id': _bytes_feature(str(record_id)),
'embedding': _float_feature(sentence_embedding.numpy())
}
return tf.train.Example(features=tf.train.Features(feature=features))
def create_embeddings(data, output_path, starting_record_id):
record_id = int(starting_record_id)
#drops labels, labes idx = (3,4)
data_seqs = np.zeros((989,11,3))
row = 0
for i in range(11,1000):
data_seqs[row] = data[i-11:i,0:3]
row = row + 1
with tf.io.TFRecordWriter(output_path) as writer:
for data_seq in data_seqs:
example = create_embedding_example(data_seq, record_id)
record_id = record_id + 1
writer.write(example.SerializeToString())
return record_id
# Persist TF.Example features containing embeddings for training data in
# TFRecord format.
create_embeddings(data, 'tmp/imdb/embeddings.tfr', 0)
##################################################################################
nsl.tools.build_graph(['tmp/imdb/embeddings.tfr'],
'tmp/imdb/graph_99.tsv',
similarity_threshold=0.99)
def create_example(data_seq, record_id):
"""Create tf.Example containing the sample's word vector, label, and ID."""
features = {
'id': _bytes_feature(str(record_id)),
'sin': _float_feature(data_seq[:, 0]),
'cos': _float_feature(data_seq[:, 1]),
'cosh': _float_feature(data_seq[:, 2]),
'label_1': _int64_feature(np.int64(data_seq[:,3])),
'label_2': _int64_feature(np.int64(data_seq[:,4])),
}
return tf.train.Example(features=tf.train.Features(feature=features))
def create_records(data, record_path, starting_record_id):
record_id = int(starting_record_id)
data_seqs = np.zeros((989, 11, 5))
row = 0
for i in range(11, 1000):
data_seqs[row] = data[i - 11:i, :]
row = row + 1
with tf.io.TFRecordWriter(record_path) as writer:
for row in range(data_seqs.shape[0]):
#for word_vector, label in zip(word_vectors, labels):
example = create_example(data_seqs[row,:], record_id)
record_id = record_id + 1
writer.write(example.SerializeToString())
return record_id
# Persist TF.Example features (word vectors and labels) for training and test
# data in TFRecord format.
next_record_id = create_records(data,'tmp/imdb/train_data.tfr', 0)
nsl.tools.pack_nbrs(
'tmp/imdb/train_data.tfr',
'',
'tmp/imdb/graph_99.tsv',
'tmp/imdb/nsl_train_data.tfr',
add_undirected_edges=True,
max_nbrs=3)
Here I have encountered problems. I have implemented several different versions but none of them works. Here is one of my versions which isn't working.
NBR_FEATURE_PREFIX = 'NL_nbr_'
NBR_WEIGHT_SUFFIX = '_weight'
class HParams(object):
"""Hyperparameters used for training."""
def __init__(self):
### dataset parameters
self.num_classes = 2
self.max_seq_length = 256
self.vocab_size = 10000
### neural graph learning parameters
self.distance_type = nsl.configs.DistanceType.L2
self.graph_regularization_multiplier = 0.1
self.num_neighbors = 2
### model architecture
self.num_embedding_dims = 16
self.num_lstm_dims = 64
self.num_fc_units = 64
### training parameters
self.train_epochs = 10
self.batch_size = 128
### eval parameters
self.eval_steps = None # All instances in the test set are evaluated.
HPARAMS = HParams()
def make_dataset(file_path, training=False):
"""Creates a `tf.data.TFRecordDataset`.
Args:
file_path: Name of the file in the `.tfrecord` format containing
`tf.train.Example` objects.
training: Boolean indicating if we are in training mode.
Returns:
An instance of `tf.data.TFRecordDataset` containing the `tf.train.Example`
objects.
"""
def pad_sequence(sequence, max_seq_length):
"""Pads the input sequence (a `tf.SparseTensor`) to `max_seq_length`."""
pad_size = tf.maximum([0], max_seq_length - tf.shape(sequence)[0])
padded = tf.concat(
[sequence.values,
tf.fill((pad_size), tf.cast(0, sequence.dtype))],
axis=0)
# The input sequence may be larger than max_seq_length. Truncate down if
# necessary.
return tf.slice(padded, [0], [max_seq_length])
def parse_example(example_proto):
"""Extracts relevant fields from the `example_proto`.
Args:
example_proto: An instance of `tf.train.Example`.
Returns:
A pair whose first value is a dictionary containing relevant features
and whose second value contains the ground truth labels.
"""
# The all features has length of 11.
feature_spec = {
'sin': tf.io.FixedLenSequenceFeature(11,tf.float32,allow_missing=True),
'cos': tf.io.FixedLenSequenceFeature(11,tf.float32,allow_missing=True),
'cosh': tf.io.FixedLenSequenceFeature(11,tf.float32,allow_missing=True),
'label_1': tf.io.FixedLenSequenceFeature(11,tf.int64,allow_missing=True),
'label_2': tf.io.FixedLenSequenceFeature(11,tf.int64,allow_missing=True),
}
# We also extract corresponding neighbor features in a similar manner to
# the features above during training.
if training:
for i in range(2):
nbr_feature_key = '{}{}_{}'.format(NBR_FEATURE_PREFIX, i, 'cos')
feature_spec[nbr_feature_key] = tf.io.FixedLenSequenceFeature([11],dtype=tf.float32,allow_missing=True)
nbr_feature_key = '{}{}_{}'.format(NBR_FEATURE_PREFIX, i, 'sin')
feature_spec[nbr_feature_key] = tf.io.FixedLenSequenceFeature([11],dtype=tf.float32,allow_missing=True)
nbr_feature_key = '{}{}_{}'.format(NBR_FEATURE_PREFIX, i, 'cosh')
feature_spec[nbr_feature_key] = tf.io.FixedLenSequenceFeature([11],dtype=tf.float32,allow_missing=True)
nbr_weight_key = '{}{}{}'.format(NBR_FEATURE_PREFIX, i,NBR_WEIGHT_SUFFIX)
# We assign a default value of 0.0 for the neighbor weight so that
# graph regularization is done on samples based on their exact number
# of neighbors. In other words, non-existent neighbors are discounted.
feature_spec[nbr_weight_key] = tf.io.FixedLenFeature(
[1], tf.float32, default_value=tf.constant([0.0]))
#features = tf.io.parse_single_example(example_proto, feature_spec)
features = tf.io.parse_single_sequence_example(example_proto, feature_spec)
# I think word padding isn't needed in this case. I might be wrong
#features['words'] = pad_sequence(features['words'], HPARAMS.max_seq_length)
#if training:
# for i in range(2):
# nbr_feature_key = '{}{}_{}'.format(NBR_FEATURE_PREFIX, i, 'words')
# features[nbr_feature_key] = pad_sequence(features[nbr_feature_key],HPARAMS.max_seq_length)
labels = tf.stack([tf.reshape(features.pop('label_1'), [11]), \
tf.reshape(features.pop('label_2'), [11])],axis=1)
return features, labels
dataset = tf.data.TFRecordDataset([file_path])
if training:
dataset = dataset.shuffle(100)
dataset = dataset.map(parse_example)
dataset = dataset.batch(64)
return dataset
train_dataset = make_dataset('tmp/imdb/nsl_train_data.tfr', True)
def make_gru_model():
input = tf.keras.layers.Input(shape=(11, 3))
x = tf.keras.layers.GaussianNoise(0.3)(input)
x = tf.keras.layers.GRU(6, reset_after=True,return_sequences=True)(x)
x = tf.keras.layers.Dropout(0.1)(x)
x = tf.keras.layers.GaussianNoise(0.3)(x)
x = tf.keras.layers.GRU(6 // 2, reset_after=True,return_sequences=True)(x)
x = tf.keras.layers.Dense(2, activation='sigmoid')(x)
model = tf.keras.Model(input, x)
return model
model = make_gru_model()
#Graph regularization
validation_fraction = 0.9
validation_size = int(validation_fraction *
int(989 / HPARAMS.batch_size))
print(validation_size)
validation_dataset = train_dataset.take(validation_size)
train_dataset = train_dataset.skip(validation_size)
graph_reg_config = nsl.configs.make_graph_reg_config(
max_neighbors=HPARAMS.num_neighbors,
multiplier=HPARAMS.graph_regularization_multiplier,
distance_type=HPARAMS.distance_type,
sum_over_axis=-1)
graph_reg_model = nsl.keras.GraphRegularization(model,graph_reg_config)
graph_reg_model.compile(
optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
graph_reg_history = graph_reg_model.fit(
train_dataset,
validation_data=validation_dataset,
epochs=HPARAMS.train_epochs,
verbose=1)
So, the problem is preparing the data correctly for Neural Network. The current code gives following error message:
ValueError: Unsupported FixedLenSequenceFeature FixedLenSequenceFeature(shape=[11], dtype=tf.float32, allow_missing=True, default_value=None).
This is my first question in Stackoverflow so I might have missed something. Please tell me if I have forgot to add some essential information to solve this problem.