I try to classification of my original records using tensorflow 1.4.0. The process is as following.
Fist: Read image and labels, and outputting "tfrecord" format into fiiles. Second: Read tf record and training
write tfrecord script is
!/usr/bin/env python3
#coding:utf-8
import argparse
import os
import random
import numpy as np
from PIL import Image
import tensorflow as tf
def make_example(label_index, image):
return tf.train.Example(features = tf.train.Features(feature={
'label_index': tf.train.Feature(int64_list=tf.train.Int64List(value=[label_index])),
'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image]))
}))
def write_tfrecord(dataset, outputfilepath):
writer = tf.python_io.TFRecordWriter(outputfilepath)
for label_of_one_hot, image in dataset:
ex = make_example(label_of_one_hot, image)
writer.write(ex.SerializeToString())
writer.close()
def importingargs():
parser = argparse.ArgumentParser("tensorflow exampe")
parser.add_argument("--datafolderpath", "-df", help="datafolderpath")
parser.add_argument("--filepath", "-f", help="filepath", required=True)
parser.add_argument("--labelfilepath", "-lf", help="label filepath")
parser.add_argument("--outputfolderpath", "-of", help="outputfolderpath of tf records")
parser.add_argument("--seed", "-s", type=int, required=False, default=0)
args = parser.parse_args()
return args.filepath, args.datafolderpath, args.labelfilepath, args.outputfolderpath, args.seed
def load_data(filepath, datafolderpath, labelfilepath):
with open(labelfilepath, "r") as rf:
labellist = [ line.strip() for line in rf.readlines() ]
with open(filepath, "r") as rf:
filepathlist = [ line.strip() for line in rf.readlines() ]
alldatasets = list()
for filepath in filepathlist:
imagefilepath = os.path.join(datafolderpath, filepath)
# image = open(imagefilepath).read()
img_obj = Image.open(imagefilepath).convert("L")
img = np.array(img_obj)
w, h = img.shape
print(w, h)
print(w*h)
img = img.reshape(w*h).tostring()
print(type(img))
filename = filepath.split(os.path.sep)[-1]
label = filename.split(".")[0].split("_")[1]
index = labellist.index(label) +1
print(index)
alldatasets.append([ index, img ])
return alldatasets
def splitdata(datasets):
random.shuffle(datasets)
train_indexes = [ 0, int(len(datasets) * 0.8 ) ]
valid_indexes = [ train_indexes[-1], int(len(datasets) * 0.9 ) ]
test_indexes = [ valid_indexes[-1], int(len(datasets)) ]
train_data = datasets[train_indexes[0]:train_indexes[1]]
valid_data = datasets[valid_indexes[0]:valid_indexes[1]]
test_data = datasets[test_indexes[0]:test_indexes[1]]
print("train num: %d" % len(train_data))
print("test num: %d" % len(test_data))
print("valid num: %d" % len(valid_data))
return train_data, valid_data, test_data
def main():
filepath, datafolderpath, labelfilepath, outputfolderpath, seed = importingargs()
random.seed(seed)
alldatasets = load_data(filepath, datafolderpath, labelfilepath)
train_data, valid_data, test_data = splitdata(alldatasets)
train_outputfilepath = os.path.join(outputfolderpath, "train.tfrecord")
valid_outputfilepath = os.path.join(outputfolderpath, "valid.tfrecord")
test_outptufilepath = os.path.join(outputfolderpath, "test.tfrecord")
write_tfrecord(train_data, train_outputfilepath)
write_tfrecord(valid_data, valid_outputfilepath)
write_tfrecord(test_data, test_outptufilepath)
if __name__ == "__main__":
main()
The load_dataset files importing train.py
#!/usr/bin/env python3
#coding:utf-8
import argparse
import os
import numpy as np
from PIL import Image
import tensorflow as tf
def read_tfrecord(inputfilepath):
print("read record")
reader = tf.TFRecordReader()
filename_que = tf.train.string_input_producer([inputfilepath])
key, value = reader.read(filename_que)
features = tf.parse_single_example(value,features = {
'label_index': tf.FixedLenFeature([], tf.string),
'image': tf.FixedLenFeature([], tf.string)
})
images = tf.decode_raw(features['image'], tf.float32)
images.set_shape([32*32])
images = tf.cast(images, tf.float32) * (1. / 255)
# images = tf.reshape(images, [-1])
labels = tf.decode_raw(features['label_index'], tf.int32)
# labels = tf.cast(features['label_index'], tf.int64)
# labels.set_shape([5])
print("call one hot")
label_index_one_hot = tf.one_hot(labels, 5)
label_index_one_hot.set_shape([5])
return images, label_index_one_hot
Training script is
import os
import random
import tensorflow as tf
import load_datasets
import datasets
import make_datasets
print("def input and output")
images = tf.placeholder(tf.float32, shape=[None, 32*32])
labels = tf.placeholder(tf.int32, shape=[None, 5])
print("def layers")
x = tf.placeholder(tf.float32, [ None, 32*32 ])
y_ = tf.placeholder(tf.float32, [None, 5 ])
# W1 = tf.Variable(tf.zeros([ 32*32, 500 ]))
# b1 = tf.Variable(tf.zeros([ 500 ]))
# W2 = tf.Variable(tf.zeros([ 500, 5 ]))
# b2 = tf.Variable(tf.zeros([ 5 ]))
print("def function")
# h1 = tf.matmul(x, W1) + b1
# y = tf.matmul(h1, W2) + b2
W = tf.Variable(tf.zeros([ 32*32, 5 ]))
b = tf.Variable(tf.zeros([ 5 ]))
y = tf.matmul(x, W) + b
print("def leraning model")
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=y))
train_step = tf.train.AdamOptimizer(0.001).minimize(cross_entropy)
correct_prediction= tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print("load train dataset")
trainfilepath = "../03tfrecords/train.tfrecord"
images, labels = load_datasets.read_tfrecord(trainfilepath)
input_queue = tf.train.slice_input_producer( [images, labels ], num_epochs=10, shuffle=False )
image_batch, label_batch = tf.train.batch( [images, labels], batch_size=10)
print("load test dataset")
testfilepath = "../03tfrecords/test.tfrecord"
test_image, test_label = load_datasets.read_tfrecord(testfilepath)
img_test_batch, label_test_batch = tf.train.batch([test_image,test_label],batch_size=16)
with tf.Session() as sess:
print("init layer value")
sess.run(tf.global_variables_initializer())
print("start training")
tf.train.start_queue_runners(sess)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
while not coord.should_stop():
for i in range(0, 10):
print("train num %d" % (i+1))
imgs, labels = sess.run([image_batch, label_batch])
sess.run(train_step, feed_dict={x:imgs, y_: labels})
imgs_test, labels_text = sess.run([img_test_batch, label_test_batch])
print(sess.run(accuracy, feed_dict={x:imgs_test, y_:labels_text}))
finally:
coord.request_stop()
coord.join(threads)
The making tfrecords works well, but in training script, an error occurs.
Traceback (most recent call last):
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 576, in merge_with
self.assert_same_rank(other)
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 621, in assert_same_rank
other))
ValueError: Shapes (?, 5) and (5,) must have the same rank
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "train.py", line 45, in <module>
images, labels = load_datasets.read_tfrecord(trainfilepath)
File "/home/omori/tensorflow_example/01src/load_datasets.py", line 30, in read_tfrecord
label_index_one_hot.set_shape([5])
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 407, in set_shape
self._shape = self._shape.merge_with(shape)
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 582, in merge_with
raise ValueError("Shapes %s and %s are not compatible" % (self, other))
ValueError: Shapes (?, 5) and (5,) are not compatible
I search the many sites but I can't get the solution. How can I do solve it?