Why is there no improvement in a categorical data time series model?

I built a simple categorical time series model to predict the next number of a random sequence, but the accuracy hardly moved even I trained it for 10000 epochs. The validation loss started to take off after a few hundred epochs. Could anyone make suggestions for improvement? Here's the model:

import os
import sys
import pandas as pd
import numpy as np
from sklearn.preprocessing import OneHotEncoder

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM


DEVICE = 'CPU'

if DEVICE == 'CPU':
    os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
else:
    os.environ['CUDA_VISIBLE_DEVICES'] = '0'
    print(tf.test.gpu_device_name())
    

TOTAL_CATALOG=4
POSSIBLE_OUTCOME_COL=4
LOOK_BACK_WINDOW=1
TRAINING_DATA_RATIO=0.8
TRAINING_EPOCHS=10000

sys.path.insert(0, '/DataScience/MyModules')
from m6data import getDrawData, series_to_supervised, Split_data, get_all_categories

def get_all_categories_local(last_combination):
    all_category = np.arange(1, last_combination+1)

    return all_category.reshape(1,all_category.shape[0])

All_categories=get_all_categories_local(TOTAL_CATALOG)

data_sequence = [1,1,2,4,2,3,1,2,3,3,4,1,2,3,4,2,2,3,1,3]

raw_df = pd.DataFrame(data_sequence, columns=['NE'])

values = raw_df.values

# 05-Apr-2022: One-Hot Encoding
oh_encoder = OneHotEncoder(categories=All_categories, sparse=False)

encoded_input = oh_encoder.fit_transform(values)
FEATURES = encoded_input.shape[1]
POSSIBLE_OUTCOME_COL = FEATURES

draw_reframe = series_to_supervised(encoded_input, LOOK_BACK_WINDOW,1)
train, test = Split_data(draw_reframe, TRAINING_DATA_RATIO)


# Total input = all possible One-Hot Encoding outcome * number of look-back samples.
ALL_INPUT = POSSIBLE_OUTCOME_COL * LOOK_BACK_WINDOW 

# split into input and outputs
train_X, train_y = train.iloc[:,:ALL_INPUT], train.iloc[:,ALL_INPUT:]
test_X, test_y = test.iloc[:,:ALL_INPUT], test.iloc[:,ALL_INPUT:]

train_X = train_X.values.reshape((train_X.shape[0], LOOK_BACK_WINDOW , FEATURES))
test_X = test_X.values.reshape((test_X.shape[0], LOOK_BACK_WINDOW, FEATURES))

print(train_X.shape, train_y.shape)
print(test_X.shape, test_y.shape)

def create_model():

    model = Sequential()

    model.add(LSTM(10, 
                   return_sequences=False,  
                   input_shape=(train_X.shape[1], train_X.shape[2]), 
                   activation='relu'
                  )
             )
    #model.add(LSTM(20))
    model.add(Dense(units=train_y.shape[1], activation='softmax'))
    
    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=0.00005),
                loss = 'categorical_crossentropy',
                metrics=['accuracy'])  

    return model

model=create_model()

history = model.fit(
    train_X, train_y,
    epochs=TRAINING_EPOCHS,
    batch_size=8,
    validation_data=(test_X, test_y),
    verbose=1,
)

Here are the plots of accuracies and losses (red=training, blue=validation).

Accuracies Losses Thank you in advance for any suggestions.

Update (13-Jun-2022) I changed my model to the following

def create_model():

model = Sequential()

model.add(LSTM(50, 
               return_sequences=True,  
               input_shape=(train_X.shape[1], train_X.shape[2]), 
               activation='relu'
              )
         )
model.add(LSTM(units=1000, kernel_regularizer=regularizers.l1(0.05), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(units=1000, kernel_regularizer=regularizers.l1(0.05), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(units=1000, kernel_regularizer=regularizers.l1(0.05), return_sequences=True))
model.add(Dropout(0.2))    
model.add(LSTM(units=1000, kernel_regularizer=regularizers.l1(0.05), activation='relu'))
model.add(Dropout(0.3))
model.add(BatchNormalization())
model.add(Dense(1000))
model.add(Dense(units=train_y.shape[1], activation='softmax'))

model.compile(optimizer = tf.keras.optimizers.SGD(learning_rate=1e-2,  nesterov=True),
              #tf.keras.optimizers.Adam(learning_rate=0.001),
            loss = 'categorical_crossentropy',
            metrics=['accuracy'])     

return model


reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
                          patience=20,min_lr=1e-10)
early_stop = EarlyStopping(monitor='loss', patience=100)

history = model.fit(
train_X, train_y,
epochs=TRAINING_EPOCHS,
batch_size=16,
validation_split=0.1,
validation_data=(test_X, test_y),
verbose=1,
shuffle=False,
callbacks=([reduce_lr], [early_stop])

Accuracy was bouncing around and Val_accuracy was zero all the way. The loss and val_loss were almost the same and dropping together.

Can anyone advise what I can do in this scenario?