Filter data in Tensorflow

user_752843 0
Viewed 4993

I have some column data in TensorFlow and I'd like to filter on one of the columns, like so:

import pandas as pd

import tensorflow.compat.v2 as tf
import tensorflow.compat.v1 as tfv1
tfv1.enable_v2_behavior()

csv_file = tf.keras.utils.get_file('heart.csv', 'https://storage.googleapis.com/applied-dl/heart.csv')

df = pd.read_csv(csv_file)
target = df.pop('target')
df['thal'] = pd.Categorical(df['thal'])
df['thal'] = df.thal.cat.codes

# Use interleave() and prefetch() to read many files concurrently.
#files = tf.data.Dataset.list_files(file_pattern=input_file_pattern, shuffle=True, seed=123456789)
#dataset = files.interleave(lambda x: tf.data.RecordIODataset(x).prefetch(100), cycle_length=8)

#Pretend I actually had some data files
dataset = tf.data.Dataset.from_tensor_slices((df.to_dict('list'), target.values))

dataset = dataset.shuffle(1000, seed=123456789)
dataset = dataset.batch(20)
#Pretend I did some parsing here
# dataset = dataset.map(parse_record, num_parallel_calls=20) 
dataset = dataset.filter(lambda x, label: x['trestbps']<135)

But this produces the error message:

ValueError: predicate return type must be convertible to a scalar boolean tensor. Was TensorSpec(shape=(None,), dtype=tf.bool, name=None).

What should I do to filter the data?

tensorflow-datasets tensorflow2.0 tensorflow python
1 Answers

This is because you apply the filter after the batch. Hence, in the lambda expression, x is a batch with shape (None,) (pass drop_reminder=True to batch to get shape of (20,)), and not a sample. To fix it, you have to call filter before batch.

There is a solution to "filter" after the batch, using a map instead. However as you can see, this has a side effect of making batches of variables size: you get a batch of 20 in the input and you remove elements not matching a specific criteria (trestbps < 135), not removing the same number of elements from each batch. Moreover this solution is performing very bad...

import timeit

import pandas as pd

import tensorflow.compat.v2 as tf
import tensorflow.compat.v1 as tfv1
tfv1.enable_v2_behavior()

def s1(ds):
    dataset = ds
    dataset = dataset.filter(lambda x, label: x['trestbps']<135)
    dataset = dataset.batch(20)
    return dataset

def s2(ds):
    dataset = ds
    dataset = dataset.batch(20)
    dataset = dataset.map(lambda x, label: (tf.nest.map_structure(lambda y: y[x['trestbps'] < 135], x), label[x['trestbps'] < 135]))
    return dataset


def base_ds():
    csv_file = tf.keras.utils.get_file('heart.csv', 'https://storage.googleapis.com/applied-dl/heart.csv')

    df = pd.read_csv(csv_file)
    target = df.pop('target')
    df['thal'] = pd.Categorical(df['thal'])
    df['thal'] = df.thal.cat.codes

    return tf.data.Dataset.from_tensor_slices((df.to_dict('list'), target.values))


def main():
    ds = base_ds()
    ds1 = s1(ds)
    ds2 = s2(ds)
    tf.print("DS_S1:", [tf.nest.map_structure(lambda x: x.shape, x) for x in ds1])
    tf.print("DS_S2:", [tf.nest.map_structure(lambda x: x.shape, x) for x in ds2])
    tf.print("Are equals?", [x for x in ds1] == [x for x in ds2])
    tf.print("Contains same elements?", [x for x in ds1.unbatch()] == [x for x in ds2.unbatch()])

    tf.print("Filter and batch:", timeit.timeit(lambda: s1(ds), number=100))
    tf.print("Batch and map:", timeit.timeit(lambda: s2(ds), number=100))

if __name__ == '__main__':
    main()

Results in:

# Tensor shapes
[...]
Are equals? False
Contains same elements? True
Filter and batch: 0.5571189750007761
Batch and map: 15.582061060000342

Kind

user_43738980
Related