I am currently participating in the Kaggle PANDA Competition and I have copied the data generator from Link.
However, this custom keras data generator is extremely inefficient due to the fact that it spends time processing images (patching and gluing images together)on every batch. To resolve this issue, I have decided to save patched and glued-together images locally, turning the processing operation into a trivial operation of loading images.
This is the previous getitem operation:
def __getitem__(self, index):
X = np.zeros((self.batch_size, self.side * self.img_size, self.side * self.img_size, 3), dtype=np.float32)
imgs_batch = self.df[index * self.batch_size : (index + 1) * self.batch_size]['image_id'].values
for i, img_name in enumerate(imgs_batch):
img_path = '{}/{}.tiff'.format(self.imgs_path, img_name)
img_patches = self.get_patches(img_path)
X[i, ] = self.glue_to_one(img_patches)
if self.mode == 'fit':
y = np.zeros((self.batch_size, self.n_classes), dtype=np.float32)
lbls_batch = self.df[index * self.batch_size : (index + 1) * self.batch_size]['isup_grade'].values
for i in range(self.batch_size):
y[i, lbls_batch[i]] = 1
return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('mode parameter error')
I created a helper function to get the image as well as its ID which would then be used to store the images locally in PNG format:
def getItemAndID(self, index):
## This is called a number equal to batch size.
ids_batch = self.df[index * self.batch_size: (index + 1) * self.batch_size]['image_id'].values
X = np.zeros((self.batch_size, self.side * self.img_size, self.side * self.img_size, 3), dtype=np.float32)
for i, img_name in enumerate(ids_batch): # Iterate over every image in batch
img_path = '{}/{}.tiff'.format(self.imgs_path, img_name)
# print(img_name)
PNGPath = os.path.join("D:\PANDA", "keras_baseline_processed_images\\") + img_name + ".png"
if not (os.path.exists(PNGPath)):
img_patches = self.get_patches(img_path)
X[i,] = self.glue_to_one(img_patches)
y = np.zeros((self.batch_size, self.n_classes), dtype=np.float32)
return X, ids_batch
This is the function I created to loop through all the batches, saving the processed images locally.
def saveBatch(datagen, BATCH_SIZE):
print("Save Batch")
for index in tqdm(range(len(datagen))):
Image, ImageIDs = datagen.getItemAndID(index)
for j in range(BATCH_SIZE):
path = os.path.join("D:\PANDA", "keras_baseline_processed_images\\") + ImageIDs[j,] + ".png"
# print(os.path.join("D:\PANDA", "keras_baseline_processed_images\\") + ImageIDs[j,] +".png")
if not (os.path.exists(path)):
# plt.imsave(path, Image[j])
image = Image[j] * 255
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
cv2.imwrite(path, image)
I finally altered the original getitem function but made a copy of the old function, known as getItemOld:
def __getitem__(self, index):
X = np.zeros((self.batch_size, self.side * self.img_size, self.side * self.img_size, 3), dtype=np.float32)
imgs_batch = self.df[index * self.batch_size : (index + 1) * self.batch_size]['image_id'].values
#for i, img_name in enumerate(imgs_batch):
# img_path = '{}/{}.tiff'.format(self.imgs_path, img_name)
# img_patches = self.get_patches(img_path)
# X[i, ] = self.glue_to_one(img_patches)
for i, img_name in enumerate(imgs_batch):
image = (cv2.imread("D:\PANDA\keras_baseline_processed_images\\" + img_name +".png"))
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
#print(type(image)) # <class 'numpy.ndarray'>
image = image/255
#print(image) # 0 - 255
#print(type(image))
#print(image) # 0 - 255
X[i,] = image
if self.mode == 'fit':
y = np.zeros((self.batch_size, self.n_classes), dtype=np.float32)
lbls_batch = self.df[index * self.batch_size : (index + 1) * self.batch_size]['isup_grade'].values
for i in range(self.batch_size):
y[i, lbls_batch[i]] = 1
return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('mode parameter error')
Both my improvement and getItemOld produce the same data type of <class 'tuple'> which contains <class 'numpy.ndarray'>. All ndarrays are equivalent and this has been verified by the code below. However, the tuple is NOT equal:
OldGetItemImage, OldGetItemLabel = train_datagen.__getItemOld__(10)
NewGetItemImage, NewGetItemLabel = train_datagen.__getitem__(10)
OldGetItem = train_datagen.__getItemOld__(10)
NewGetItem = train_datagen.__getitem__(10)
print(np.array_equal(OldGetItemImage, NewGetItemImage))
print(np.array_equal(OldGetItemLabel, NewGetItemLabel))
print(np.array_equal(OldGetItem, NewGetItem))
The above comparison code produces the following result:
True
True
False
Due to this, the data generator has significantly lower accuracy compared to the original slower variant and I have no idea why.