What is the difference between and ?

Question

I am beginner in tensorflow and studying how to use tfrecord dataset.

is there any difference between and ??

version of tensorflow is 2.3.0

for Mnist example

import numpy as np
import tensorflow as tf
from tensorflow.keras.datasets import mnist

def feature_float_list(l):
    return tf.train.Feature(float_list=tf.train.FloatList(value=l))

def record2example(r_x, r_y):
    return tf.train.Example(features=tf.train.Features(feature={
        "x": feature_float_list(r_x),
        "y": feature_float_list(r_y)
    }))

filename_test  = "test.tfrecords"

# Data of MNIST to tfrecord file
_, (x_test, y_test) = mnist.load_data()
print("x_test: ", x_test.shape)  # x_test    :  (10000, 28, 28)
print("y_test: ", y_test.shape)  # y_test    :  (10000,)
x_test  = x_test.reshape((-1, 28*28)).astype("float32") / 255.0
y_test  = y_test.reshape((-1, 1)).astype("float32")

with tf.io.TFRecordWriter(filename_test) as writer:
    for r_x, r_y in zip(x_test, y_test):
        ex = record2example(r_x, r_y)
        writer.write(ex.SerializeToString())

#output
#x_test:(10000, 28, 28)
#y_test:(10000,)

and,load tfrecord dataset like below


feature_dim=784 

def parse_example(example):
    features = tf.io.parse_single_example(example, features={
        "x": tf.io.FixedLenFeature([feature_dim], dtype=tf.float32),
        "y": tf.io.FixedLenFeature([], dtype=tf.float32)
    })
    x = features["x"]
    y = features["y"]
    return x, y

#-----tf.io.parse_single_example -----ver-----

def create_dataset_1():
    
    dataset = tf.data.TFRecordDataset(["test.tfrecords"]).map(parse_example)

    dataset = dataset.repeat()

    buffer_size=10
    dataset = dataset.shuffle(buffer_size)
    
    #batchsize
    batch_size=10
    dataset = dataset.batch(batch_size)
    
    # set iterator
    iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)
    image, label = iterator.get_next()

    return image, label

image,label=create_dataset_1()
print(image.shape)
print(label.shape)

#(10, 784)
#(10,)

#-----tf.data.experimental.parse_example_dataset -----ver-----

def dict2tuple(feature):
    return feature["x"], feature["y"]

def create_dataset_2():

    feature_dim=784
    dataset = tf.data.TFRecordDataset(["test.tfrecords"]).batch(100).apply(tf.data.experimental.parse_example_dataset(
        {
              "x": tf.io.FixedLenFeature([feature_dim], dtype=tf.float32),
              "y": tf.io.FixedLenFeature([], dtype=tf.float32)
          })).map(dict2tuple)
        
    #set repeat and shuffle
    dataset = dataset.repeat()
    
    dataset = dataset.shuffle(10)
    
    # batchsize
    #batch_size=10
    #dataset = dataset.batch(batch_size)
    
    #iterator
    iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)
    
    image, label = iterator.get_next()

    return image, label

image,label=create_dataset_2()
print(image.shape)
print(label.shape)

#(100, 784)
#(100,)

I tried to look it up but,could not find this,think both outputs is the same...., which should I use and ?

user_14290697 · Answer

There is not much difference in both cases.

In tf.data.experimental.parse_example_dataset the below apply function is retuned.

def _apply_fn(dataset):
    """Function from Dataset to Dataset that applies the transformation."""
    out_dataset = _ParseExampleDataset(dataset, features, num_parallel_calls,
                                       deterministic)
    if any(
        isinstance(feature, parsing_ops.SparseFeature) or
        (isinstance(feature, parsing_ops.RaggedFeature) and feature.partitions)
        for feature in features.values()):
      # pylint: disable=protected-access
      # pylint: disable=g-long-lambda
      out_dataset = out_dataset.map(
          lambda x: parsing_ops._construct_tensors_for_composite_features(
              features, x),
          num_parallel_calls=num_parallel_calls)
    return out_dataset

  return _apply_fn

Where in tf.io.parse_single_example Parses a serialized Example protos given in serialized which is similar to tf.io.parse_example, except tf.io.parse_example is serialized in batches.

You can use tf.io.parse_example for performance advantage over batching samples.

What is the difference between <tf.io.parse_single_example> and <tf.data.experimental.parse_example_dataset>?

for Mnist example

and,load tfrecord dataset like below

1 Answers