Model and Weights do not load from checkpoint

First, the code won't run if you don't already have the weights/model saved. So I commented out the below lines and ran the script to generate the files for the first time.

    checkpoint = tf.train.get_checkpoint_state(".")
    print('checkpoint:', checkpoint)
    if checkpoint and checkpoint.model_checkpoint_path:
        dqn_solver.model = tf.keras.models.load_model('cartpole.h5')
        dqn_solver.model.load_weights('cartpole_weights.h5')

Note I also modified the above code - there were some syntax errors before. In particular, this line in your post

dqn_solver.model = model.load_weights('cartpole_weights.h5')

is probably what was causing the problem, because the model.load_weights('file') method mutates model (as opposed to returning the model).

I then tested that the model weights were being saved/loaded correctly. To do this, you can do

dqn_solver = DQNSolver(observation_space, action_space)
dqn_solver.model.trainable_variables

To see the (randomly initialized) weights for when the model first gets made. Then you can load the weights with either

dqn_solver.model = tf.keras.models.load_model('cartpole.h5')

or

dqn_solver.model.load_weights('cartpole_weights.h5')

and then you can again view the trainable_variables to make sure that they are different than the initial weights, and that they are equivalent.

When you save a model, it saves the full architecture - the exact configuration of layers. When you save the weights, it just saves all the list of tensors that you can see with trainable_variables. Note that when you load_weights, it needs to be loaded into the exact architecture the weights are for, otherwise it won't work correctly. So if you changed the model arcthiecture in DQNSolver, and then tried to load_weights for the old model, it's not going to work right. If you load_model it will reset the model to exactly how the architecture was, and also set the weights.

edit - entire modified script

## Slightly modified from the following repository - https://github.com/gsurma/cartpole

from __future__ import absolute_import, division, print_function, unicode_literals

import os
import random
import gym
import numpy as np
import tensorflow as tf

from collections import deque
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint


ENV_NAME = "CartPole-v1"

GAMMA = 0.95
LEARNING_RATE = 0.001

MEMORY_SIZE = 1000000
BATCH_SIZE = 20

EXPLORATION_MAX = 1.0
EXPLORATION_MIN = 0.01
EXPLORATION_DECAY = 0.995

checkpoint_path = "training_1/cp.ckpt"


class DQNSolver:

    def __init__(self, observation_space, action_space):
        # save_dir = args.save_dir
        # self.save_dir = save_dir
        # if not os.path.exists(save_dir):
        #     os.makedirs(save_dir)
        self.exploration_rate = EXPLORATION_MAX

        self.action_space = action_space
        self.memory = deque(maxlen=MEMORY_SIZE)

        self.model = Sequential()
        self.model.add(Dense(24, input_shape=(observation_space,), activation="relu"))
        self.model.add(Dense(24, activation="relu"))
        self.model.add(Dense(self.action_space, activation="linear"))
        self.model.compile(loss="mse", optimizer=Adam(lr=LEARNING_RATE))

    def remember(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))

    def act(self, state):
        if np.random.rand() < self.exploration_rate:
            return random.randrange(self.action_space)
        q_values = self.model.predict(state)
        return np.argmax(q_values[0])

    def experience_replay(self):
        if len(self.memory) < BATCH_SIZE:
            return
        batch = random.sample(self.memory, BATCH_SIZE)
        for state, action, reward, state_next, terminal in batch:
            q_update = reward
            if not terminal:
                q_update = (reward + GAMMA * np.amax(self.model.predict(state_next)[0]))
            q_values = self.model.predict(state)
            q_values[0][action] = q_update
            self.model.fit(state, q_values, verbose=0)
        self.exploration_rate *= EXPLORATION_DECAY
        self.exploration_rate = max(EXPLORATION_MIN, self.exploration_rate)


def cartpole():
    env = gym.make(ENV_NAME)
    #score_logger = ScoreLogger(ENV_NAME)
    observation_space = env.observation_space.shape[0]
    action_space = env.action_space.n
    dqn_solver = DQNSolver(observation_space, action_space)

    # checkpoint = tf.train.get_checkpoint_state(".")
    # print('checkpoint:', checkpoint)
    # if checkpoint and checkpoint.model_checkpoint_path:
    #     dqn_solver.model = tf.keras.models.load_model('cartpole.h5')
    #     dqn_solver.model.load_weights('cartpole_weights.h5')
    run = 0
    i = 0
    while i<2:
        i = i + 1
        #total = 0
        run += 1
        state = env.reset()
        state = np.reshape(state, [1, observation_space])
        step = 0
        while True:
            step += 1
            #env.render()
            action = dqn_solver.act(state)
            state_next, reward, terminal, info = env.step(action)
            #total += reward
            reward = reward if not terminal else -reward
            state_next = np.reshape(state_next, [1, observation_space])
            dqn_solver.remember(state, action, reward, state_next, terminal)
            state = state_next
            dqn_solver.model.save('cartpole.h5')
            dqn_solver.model.save_weights('cartpole_weights.h5')
            if terminal:
                print("Run: " + str(run) + ", exploration: " + str(dqn_solver.exploration_rate) + ", score: " + str(step))
                #score_logger.add_score(step, run)
                break
            dqn_solver.experience_replay()


if __name__ == "__main__":
    cartpole()

#%%  to load saved results
env = gym.make(ENV_NAME)
#score_logger = ScoreLogger(ENV_NAME)
observation_space = env.observation_space.shape[0]
action_space = env.action_space.n
dqn_solver = DQNSolver(observation_space, action_space)

dqn_solver.model = tf.keras.models.load_model('cartpole.h5')  # or
dqn_solver.model.load_weights('cartpole_weights.h5')