After days trying to make one RL agent, I finally succeeded in creating its experience, but when I try to train it I get this error. I've tried all I could: different experience, changed step params... I am just out of ideas.
import pyxinput
import time
import cv2
from PIL import ImageGrab
import numpy as np
import keyboard
import tensorflow
import tf_agents
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import torch
#from tf_agents.networks import actor_distribution_networ
from tf_agents.policies import random_py_policy
Tensod_spec = tf_agents.specs.BoundedArraySpec(
(15,),
dtype=np.float32,
name="XimputSpecs",
minimum=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
maximum=[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
)
Tensod_spec2 = tf_agents.specs.TensorSpec(
[440, 600, 1], dtype=tf.int32, name="ScreenSpecs"
)
Tensor_reward_spe = tf_agents.specs.TensorSpec(
[1, 1], dtype=tf.int32, name="Reward"
)
FromEnv = tf_agents.specs.BoundedTensorSpec(
shape=(440, 600, 1),
dtype='uint8',
name='observation',
minimum=0,
maximum=255
)
FromEnv2 = tf_agents.specs.BoundedTensorSpec(
shape=(1, 440, 600, 1),
dtype=tf.int32,
name='observation',
minimum=0,
maximum=255
)
fullscreen = [110, 130, 710, 570]
screenpil = ImageGrab.grab(bbox=fullscreen)
showprint = np.array(screenpil)
grayscreen = cv2.cvtColor(showprint, cv2.COLOR_BGR2GRAY)
screenrect = cv2.cvtColor(grayscreen, cv2.COLOR_GRAY2BGR)
grayscreen = grayscreen.reshape(440, 600, 1)
time_step_spec2 = tf_agents.trajectories.time_step.time_step_spec(
observation_spec=FromEnv,
#reward_spec = Tensor_reward_spec
)
time_step_spec = tf_agents.trajectories.time_step.time_step_spec(
observation_spec=FromEnv,
#reward_spec = Tensor_reward_spec
)
actor_net = tf_agents.networks.actor_distribution_network.ActorDistributionNetwork(
input_tensor_spec=FromEnv,
output_tensor_spec=tf_agents.specs.tensor_spec.from_spec(Tensod_spec),
activation_fn='relu',
#conv_layer_params=[(25, 40, 2)],
fc_layer_params=(50, 25, 15),
#dtype='int32'
)
print(actor_net)
train_step_counter = tf.dtypes.cast(1, tf.int32)
optimizer = tf.keras.optimizers.Adam(learning_rate=0.003)
tf_agent = tf_agents.agents.ReinforceAgent(
time_step_spec=time_step_spec,
action_spec=tf_agents.specs.tensor_spec.from_spec(Tensod_spec),
actor_network=actor_net,
optimizer=optimizer,
normalize_returns=True,
#train_step_counter=tf.Variable(1, name="global_step")
)
tf_agent.initialize()
grayscreen2 = grayscreen
grayscreen2 = grayscreen2.reshape(1, 440, 600, 1)
time_step2 = tf_agents.trajectories.time_step.TimeStep(
step_type=tf_agents.trajectories.time_step.StepType.FIRST,
reward=tf.dtypes.cast(1, tf.float32),
discount=tf.dtypes.cast(1, tf.float32),
observation=grayscreen2
)
policy_state = tf_agent.policy.get_initial_state(batch_size=1)
policy_step = tf_agent.policy.action(time_step2, policy_state)
print(policy_step)
observe = time_step2.observation
#print(observe.dtype)
#observe = observe.astype(int)
#print(observe.shape)
experience = tf_agents.trajectories.trajectory.Trajectory(
action=tf.compat.v2.Variable([
tf.compat.v2.Variable(policy_step.action),
tf.compat.v2.Variable(policy_step.action),
tf.compat.v2.Variable(policy_step.action)
]),
reward=tf.compat.v2.Variable([[
tf.compat.v2.Variable(time_step2.reward),
tf.compat.v2.Variable(time_step2.reward),
tf.compat.v2.Variable(time_step2.reward)
]]),
step_type=tf.compat.v2.Variable([[
tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.FIRST),
tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.MID),
tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.LAST)
]]),
observation=tf.compat.v2.Variable([
tf.compat.v2.Variable(observe),
tf.compat.v2.Variable(observe),
tf.compat.v2.Variable(observe)
]),
policy_info=tf_agent.policy.info_spec,
next_step_type=tf.compat.v2.Variable([[
tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.MID),
tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.LAST),
tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.LAST)
]]),
discount=tf.compat.v2.Variable([[
tf.dtypes.cast(1, tf.float32),
tf.dtypes.cast(1, tf.float32),
tf.dtypes.cast(1, tf.float32)
]]),
)
train_loss = tf_agent.train(experience)
print(train_loss)
And I get this error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-15-4dd3966a32b6> in <module>
1 #
----> 2 train_loss = tf_agent.train(experience)
3 print(train_loss)
~\AppData\Local\Programs\Python\Python38\lib\site-packages\tf_agents\agents\tf_agent.py in train(self, experience, weights, **kwargs)
516
517 if self._enable_functions:
--> 518 loss_info = self._train_fn(
519 experience=experience, weights=weights, **kwargs)
520 else:
~\AppData\Local\Programs\Python\Python38\lib\site-packages\tf_agents\utils\common.py in with_check_resource_vars(*fn_args, **fn_kwargs)
183 # We're either in eager mode or in tf.function mode (no in-between); so
184 # autodep-like behavior is already expected of fn.
--> 185 return fn(*fn_args, **fn_kwargs)
186 if not resource_variables_enabled():
187 raise RuntimeError(MISSING_RESOURCE_VARIABLES_ERROR)
~\AppData\Local\Programs\Python\Python38\lib\site-packages\tf_agents\agents\reinforce\reinforce_agent.py in _train(self, experience, weights)
286 self.train_step_counter)
287
--> 288 self._optimizer.apply_gradients(
289 grads_and_vars, global_step=0)
290
TypeError: apply_gradients() got an unexpected keyword argument 'global_step'
What is this global step, and where is this error coming from? Why can't I train my agent?
Specs:
- Python 3.8
- TensorFlow 2.4 (GPU and non-GPU)
- Windows 10 / ubuntu
If you need more info, please let me know.
EDIT: Tried other agents they run fine and i posted this ISUE on Tensor GIT:https://github.com/tensorflow/tensorflow/issues/48424 If anyone has the same problem in the future