import random
import gym
import numpy as np
from collections import deque
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam
import os
env = gym.make('CartPole-v1')
state_size = env.observation_space.shape[0]
state_size
action_size = env.action_space.n
action_size
batch_size = 32
n_episodes = 1001
output_dir = 'model_output/cartpole/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
class DQNAgent:
def __init__(self, state_size, action_size):
self.state_size = state_size
self.action_size = action_size
self.memory = deque(maxlen=2000) # double-ended queue; acts like list, but elements can be added/removed from either end
self.gamma = 0.95 # decay or discount rate: enables agent to take into account future actions in addition to the immediate ones, but discounted at this rate
self.epsilon = 1.0 # exploration rate: how much to act randomly; more initially than later due to epsilon decay
self.epsilon_decay = 0.995 # decrease number of random explorations as the agent's performance (hopefully) improves over time
self.epsilon_min = 0.01 # minimum amount of random exploration permitted
self.learning_rate = 0.001 # rate at which NN adjusts models parameters via SGD to reduce cost
self.model = self._build_model() # private method
def _build_model(self):
# neural net to approximate Q-value function:
model = Sequential()
model.add(Dense(24, input_dim=self.state_size, activation='relu')) # 1st hidden layer; states as input
model.add(Dense(24, activation='relu')) # 2nd hidden layer
model.add(Dense(self.action_size, activation='linear')) # 2 actions, so 2 output neurons: 0 and 1 (L/R)
model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))
return model
def remember(self, state, action, reward, next_state, done):
self.memory.append(
(state, action, reward, next_state, done)) # list of previous experiences, enabling re-training later
def act(self, state):
if np.random.rand() <= self.epsilon: # if acting randomly, take random action
return random.randrange(self.action_size)
act_values = self.model.predict(state) # if not acting randomly, predict reward value based on current state
return np.argmax(act_values[0]) # pick the action that will give the highest reward (i.e., go left or right?)
def replay(self, batch_size): # method that trains NN with experiences sampled from memory
minibatch = random.sample(self.memory, batch_size) # sample a minibatch from memory
for state, action, reward, next_state, done in minibatch: # extract data for each minibatch sample
target = reward # if done (boolean whether game ended or not, i.e., whether final state or not), then target = reward
if not done: # if not done, then predict future discounted reward
target = (reward + self.gamma * np.amax(self.model.predict(next_state)[0])) # (maximum target Q based on future action a')
target_f = self.model.predict(state) # approximately map current state to future discounted reward
target_f[0][action] = target
self.model.fit(state, target_f, epochs=1, verbose=0) # single epoch of training with x=state, y=target_f; fit decreases loss btwn target_f and y_hat
if self.epsilon > self.epsilon_min:
self.epsilon *= self.epsilon_decay
def load(self, name):
self.model.load_weights(name)
def save(self, name):
self.model.save_weights(name)
agent = DQNAgent(state_size, action_size)
done = False
for e in range(n_episodes): # iterate over new episodes of the game
state = env.reset() # reset state at start of each new episode of the game
state = np.reshape(state, [1, state_size])
for time in range(5000): # time represents a frame of the game; goal is to keep pole upright as long as possible up to range, e.g., 500 or 5000 timesteps
env.render()
action = agent.act(state) # action is either 0 or 1 (move cart left or right); decide on one or other here
next_state, reward, done, _ = env.step(action) # agent interacts with env, gets feedback; 4 state data points, e.g., pole angle, cart position
reward = reward if not done else -10 # reward +1 for each additional frame with pole upright
next_state = np.reshape(next_state, [1, state_size])
agent.remember(state, action, reward, next_state, done) # remember the previous timestep's state, actions, reward, etc.
state = next_state # set "current state" for upcoming iteration to the current next state
if done: # episode ends if agent drops pole or we reach timestep 5000
print("episode: {}/{}, score: {}, e: {:.2}".format(e, n_episodes, time, agent.epsilon))
break # exit loop
if len(agent.memory) > batch_size:
agent.replay(batch_size) # train the agent by replaying the experiences of the episode
if e % 50 == 0:
agent.save(output_dir + "weights_" + '{:04d}'.format(e) + ".hdf5")
I follow this video on OpenAI Gymand and wrote the code one-to-one. https://www.youtube.com/watch?v=OYhFoMySoVs&t=2444s
Unfortunately I keep getting this error message:
ValueError: cannot reshape array of size 2 into shape (1,4)
I went through the code again and again and compared it with the code from the video and it is identical.
I also googled the error message and went through all the forums, but still couldn't fix the error.
The following is the entire error message:
: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cudart64_110.dll'; dlerror: cudart64_110.dll not found
: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cudart64_110.dll'; dlerror: cudart64_110.dll not found
: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cublas64_11.dll'; dlerror: cublas64_11.dll not found
: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cublasLt64_11.dll'; dlerror: cublasLt64_11.dll not found
: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cufft64_10.dll'; dlerror: cufft64_10.dll not found
: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'curand64_10.dll'; dlerror: curand64_10.dll not found
: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cusolver64_11.dll'; dlerror: cusolver64_11.dll not found
: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cusparse64_11.dll'; dlerror: cusparse64_11.dll not found
: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cudnn64_8.dll'; dlerror: cudnn64_8.dll not found
: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1934] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX AVX2
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
D:\SSD\venv\lib\site-packages\keras\optimizers\optimizer_v2\adam.py:114: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
super().__init__(name, **kwargs)
D:\SSD\venv\lib\site-packages\numpy\core\fromnumeric.py:43: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
result = getattr(asarray(obj), method)(*args, **kwds)
Traceback (most recent call last):
File "D:\SSD, line 84, in <module>
state = np.reshape(state, [1, state_size])
File "<__array_function__ internals>", line 180, in reshape
File "D:\SSD\venv\lib\site-packages\numpy\core\fromnumeric.py", line 298, in reshape
return _wrapfunc(a, 'reshape', newshape, order=order)
File "D:\SSD\venv\lib\site-packages\numpy\core\fromnumeric.py", line 54, in _wrapfunc
return _wrapit(obj, method, *args, **kwds)
File "D:\SSD\venv\lib\site-packages\numpy\core\fromnumeric.py", line 43, in _wrapit
result = getattr(asarray(obj), method)(*args, **kwds)
ValueError: cannot reshape array of size 2 into shape (1,4)
Process finished with exit code 1