How to use black and white images in keras CNN?

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Activation, Dense, Flatten, BatchNormalization, Conv2D, MaxPool2D, Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import os
import matplotlib.pyplot as plt
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

os.chdir('C:/Users/dancu/PycharmProjects/firstCNN/data/ad-vs-cn')

physical_devices = tf.config.experimental.list_physical_devices('GPU')
print("Num GPUs Available: ", len(physical_devices))
tf.config.experimental.set_memory_growth(physical_devices[0], True)

train_path = "C:/Users/dancu/PycharmProjects/firstCNN\data/ad-vs-cn/train"
test_path = "C:/Users/dancu/PycharmProjects/firstCNN\data/ad-vs-cn/test"
valid_path = "C:/Users/dancu/PycharmProjects/firstCNN\data/ad-vs-cn/valid"

train_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.vgg16.preprocess_input) \
    .flow_from_directory(directory=train_path, target_size=(256,256), classes=['cn', 'ad'], batch_size=10, color_mode="rgb")
valid_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.vgg16.preprocess_input) \
    .flow_from_directory(directory=valid_path, target_size=(256,256), classes=['cn', 'ad'], batch_size=10, color_mode="rgb")
test_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.vgg16.preprocess_input) \
    .flow_from_directory(directory=test_path, target_size=(256,256), classes=['cn', 'ad'], batch_size=10, color_mode="rgb", shuffle=False)


# def plotImages(images_arr):
#     fig, axes = plt.subplots(1, 10, figsize=(20,20))
#     axes = axes.flatten()
#     for img, ax in zip( images_arr, axes):
#         ax.imshow(img)
#         ax.axis('off')
#     plt.tight_layout()
#     plt.show()
#
#
# imgs, labels = next(train_batches)
# plotImages(imgs)

model = Sequential([
    Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = 'same', input_shape=(256,256,3)),
    MaxPool2D(pool_size=(2, 2), strides=2),
    Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding = 'same'),
    MaxPool2D(pool_size=(2, 2), strides=2),
    Flatten(),
    Dense(units=2, activation='softmax')
])

#print(model.summary())

model.compile(optimizer=Adam(learning_rate=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])

model.fit(x=train_batches,
    steps_per_epoch=len(train_batches),
    validation_data=valid_batches,
    validation_steps=len(valid_batches),
    epochs=10,
    verbose=2
)

This code runs perfectly fine, but the images I'm using are actually grayscale, so I'm getting terrible output accuracies due to the way the images appear. When I change the color_mode to "grayscale" I get the following error:

Traceback (most recent call last):
  File "C:/Users/dancu/PycharmProjects/firstCNN/finalData.py", line 56, in <module>
    model.fit(x=train_batches,
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\tensorflow\python\keras\engine\training.py", line 66, in _method_wrapper
    return method(self, *args, **kwargs)
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\tensorflow\python\keras\engine\training.py", line 802, in fit
    data_handler = data_adapter.DataHandler(
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1100, in __init__
    self._adapter = adapter_cls(
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 901, in __init__
    super(KerasSequenceAdapter, self).__init__(
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 772, in __init__
    peek, x = self._peek_and_restore(x)
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 912, in _peek_and_restore
    return x[0], x
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\keras_preprocessing\image\iterator.py", line 65, in __getitem__
    return self._get_batches_of_transformed_samples(index_array)
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\keras_preprocessing\image\iterator.py", line 239, in _get_batches_of_transformed_samples
    x = self.image_data_generator.standardize(x)
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\keras_preprocessing\image\image_data_generator.py", line 708, in standardize
    x = self.preprocessing_function(x)
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\tensorflow\python\keras\applications\vgg16.py", line 232, in preprocess_input
    return imagenet_utils.preprocess_input(
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\tensorflow\python\keras\applications\imagenet_utils.py", line 106, in preprocess_input
    return _preprocess_numpy_input(
  File "C:\Users\dancu\PycharmProjects\firstCNN\venv\lib\site-packages\tensorflow\python\keras\applications\imagenet_utils.py", line 223, in _preprocess_numpy_input
    x[..., 1] -= mean[1]
IndexError: index 1 is out of bounds for axis 2 with size 1

Process finished with exit code 1

I also changed the input_shape of the Conv2D layer to only have 1 channel rather than 3, but the same error occurs.

Could anyone assist me with this? Thank you!