error when trying to run my tensorflow code

There is few problems.

1. you return flat array with 786 values but it needs 2D array 28x28 - and this makes problem with axis=1 in normalize - you have to use self.array instead of self.full

2. you create array 28x28 but predict needs 1x28x28 - so it needs [self.array] instead of `self.array

result = nn.new_model.predict(tf.keras.utils.normalize(np.array([Sc.array]), axis=1))    

3. it returns probability for all digits and you have to use np.argmax() to get digit with the biggest probability. Because predict() can get list with many images so it returns list with many results - even if you check single image - and it needs [0] to check result for first image.

digit = np.argmax(result[0])

4. I think there is problem with data in array. You keep it as x,y but numpy uses y,x (row, column) (and the same is with matrix in math). It needs self.array[y][x] = 1 instea self.array[x][y] = 1. EDIT: I added plt.imshow(), cv2.imshow() to confirm it. Using x,y it gives rotated image.

5. you import main and this runs all code in main and it trains model again at every start. You don't have to import main but only use load_model().

6. on my computer sometimes mouse.get_pos() gives value 280 and this gives array[...][28], and this raises error because array has only [27]. It needs to check x, y and convert 280 to 279 (or later convert 28 to 27)

7. you setArr() is wrong. You append the same list self.shortArr to self.array but (this is popular problem in Python) it doesn't put copy of array but it puts reference to the same array - and finally when you change one value in row then it change them in all rows.

EDIT:

8. mnist has images which use values 0...255 so maybe you should also use self.array[y][x] = 255 instead of self.array[y][x] = 1

My full working code:

screen.py

EDIT: I added right mouse to clear single pixel(s)

EDIT: I added plt.imshow(), cv2.imshow()

import pygame
import tensorflow as tf
import numpy as np

class Screen:
    
    def __init__(self):
        pygame.init()

        #self.screen = pygame.display.set_mode((28,28),pygame.FULLSCREEN)
        self.screen = pygame.display.set_mode((280,280))
        self.bg_color = (250, 250,250)
        
        self.array = []
        self.set_arr()

        self.ok = False    # exit with prediction
        self.done = False
        #self.button = False

    def run(self):
        self.screen.fill(self.bg_color)
        while not self.done:
            pygame.display.flip()
            self._check_events()
            self.draw()
        pygame.quit()  # some systems need it to close window
        return self.ok
    
    def _check_events(self):
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                # end program without OK
                self.done = True
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_ESCAPE:
                    # end program without OK
                    self.done = True
                if event.key == pygame.K_d:
                    # end program with OK
                    self.ok = True
                    self.done = True
                if event.key == pygame.K_c:
                    # clear array and screen
                    self.set_arr()
                    self.screen.fill(self.bg_color)
            #if event.type == pygame.MOUSEBUTTONDOWN:
            #    #print("mouseDown")
            #    self.button = True
            #elif event.type == pygame.MOUSEBUTTONUP:
            #    self.button = False

    def set_arr(self):
        """Create empty array 2D for image."""
        self.array = [[0 for x in range(28)] for y in range(28)]

    def draw(self):
        buttons = pygame.mouse.get_pressed()
        if buttons[0] or buttons[2]:  # left or right button
        #if self.button:
            x0, y0 = pygame.mouse.get_pos()
            
            if x0 >= 280:
                x0 = 279
            if y0 >= 280:
                y0 = 279
                
            x = int(round(x0/10)*10)
            y = int(round(y0/10)*10)

            try:
                #self.array[int(x)//10][int(y)//10] = 255
                if buttons[0]:  # left draw
                    self.array[int(y)//10][int(x)//10] = 255
                    pygame.draw.rect(self.screen, (0,0,0), pygame.Rect(x, y, 10, 10))
                if buttons[2]:  # right clear
                    self.array[int(y)//10][int(x)//10] = 0                
                    pygame.draw.rect(self.screen, (255,255,255), pygame.Rect(x, y, 10, 10))
            except Exception as ex:
                print('Exception:', ex)
                print('Debug:', x0, y0, x, y, int(x)//10, int(y)//10)

            #print("draw")

if __name__ == '__main__':
    print('loading model ...')
    model = tf.keras.models.load_model('num_reader.model')

    print('starting screen')
    screen = Screen()
    ok = screen.run()
    
    if not ok:
        print('finish without prediction')
    else:
        image = np.array(screen.array, dtype='uint8')
        print('image.shape:', image.shape)
        
        print('--- show image ---')
        
        import matplotlib.pyplot as plt
        plt.imshow(image, cmap=plt.get_cmap('gray'))        
        plt.show()
        
        #import cv2
        #cv2.imshow('image', image)        
        #cv2.waitKey(0)
        
        print('--- predict as is ---')
        
        arr = np.array([image])
        print('arr.shape:', arr.shape)
        #print(arr[:,:2])
        results = model.predict(arr)
        print('results:', results)
        digit = np.argmax(results[0])
        print('digit:', digit)

        print('--- predict normalized ---')

        arr = tf.keras.utils.normalize(np.array([image]), axis=1)
        print('arr.shape:', arr.shape)
        #print(arr[:,:2])
        results = model.predict(arr)
        print('results:', results)
        digit = np.argmax(results[0])
        print('digit:', digit)

train_model.py

import tensorflow as tf

mnist = tf.keras.datasets.mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train = tf.keras.utils.normalize(x_train, axis=1)
x_test = tf.keras.utils.normalize(x_test, axis=1)

model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax))

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

model.fit(x_train, y_train, epochs=3)

val_loss, val_acc = model.evaluate(x_test, y_test)
print('val_loss:', val_loss)
print('val_acc :', val_acc)

#predictions = model.predict([x_test])
#print(predictions)

model.save("num_reader.model")