My DQN implementation is as follows:
class DQNAgent(nn.Module):
def __init__(self, state_size, action_size):
super().__init__()
self.state_size = state_size
self.action_size = action_size
self.memory = deque(maxlen=500)
print(f"self.memory {type(self.memory)}")
self.gamma = 0.95 # discount rate
self.epsilon = 1.0 # exploration rate
self.epsilon_min = 0.01
self.epsilon_decay = 0.995
self.learning_rate = 0.001
self.model = self._build_model()
def _build_model(self):
# Neural Net for Deep-Q learning Model
model = Sequential()
model.add(Dense(24, input_dim=100, activation='relu'))
model.add(Dense(24, activation='relu'))
model.add(Dense(self.action_size, activation='linear'))
model.compile(loss='mse',
optimizer=SGD(lr=self.learning_rate))
return model
def memorize(self, state, action, reward, next_state, done):
self.memory.append((state, action, reward, next_state, done))
print(f"Memory length {len(self.memory)} size {sys.getsizeof(self.memory)}")
def act(self, state):
if np.random.rand() <= self.epsilon:
return random.randrange(self.action_size)
act_values = self.model.predict(state)
return np.argmax(act_values[0]) # returns action
def replay(self, batch_size, slot_number):
minibatch = random.sample(self.memory, batch_size, )
print(f"Memory minibatch {len(minibatch)} size {sys.getsizeof(minibatch)}")
for state, action, reward, next_state, done in minibatch:
target = reward
if not done:
target = (reward + self.gamma *
np.amax(self.model.predict(next_state)[0]))
target_f = self.model.predict(state)
target_f[0][action] = target
self.model.fit(state, target_f, epochs=1, verbose=0)
# print(f"Model size {sys.getsizeof(self.model)}")
if self.epsilon > self.epsilon_min:
self.epsilon *= self.epsilon_decay
writer.add_scalar("Epsilon/slot_number", self.epsilon, slot_number)
# def load(self, name):
# self.model.load_weights(name)
# def save(self, name):
# torch.save(name)
# print("jamoo")
Here is how I call this dqn implementation:
def run_code():
global state, i, fileNumber, action, reward
SLOTS = 500
EPISODES = 100
for e in range(EPISODES):
# print(f"Episode {e}")
episode_reward = 0
state = env.reset()
# print(f"state {state}")
state = np.reshape(state, [1, 100])
for slot_number in range(SLOTS):
# print(slot_number)
slot = slot_number + 1
for i in range(0, 25):
fileNumber = i + 1
requests_for_file = Req_Count_Slot_V1[i][slot_number]
if requests_for_file > 0: file_request_count[f"file-{fileNumber}"] = file_request_count[
f"file-{fileNumber}"] + 1
if slot % 100 == 0 and slot > 0:
cache_files = []
top_5_files = []
updated_cache_files = []
for v in range(len(Veh1_State)):
fn = v + 1
if Veh1_State[v][0] == 1:
cache_files.append(fn)
# print(cache_files)
# print(len(cache_files))
sorted_list = (sorted(file_request_count.items(), key=operator.itemgetter(1), reverse=True))
for var in range(len(sorted_list)):
item = sorted_list[var]
file_name = int(item[0].split("-")[1])
request_count = int(item[1])
if var < 5:
top_5_files.append(file_name)
# print(f" Top {file_name} {request_count}")
if file_name in cache_files:
# print("file " + str(file_name) + " : " + str(request_count))
for v in range(len(top_5_files)):
f = top_5_files[v]
if file_name == f:
# print(f"{file_name} {request_count}")
updated_cache_files.append(file_name)
else:
updated_cache_files.append(f)
updated_cache_files = set(updated_cache_files)
# print(f"{slot} Set {(updated_cache_files)} {cache_files}")
for v in range(len(Veh1_State)):
f_number = v + 1
if f_number in updated_cache_files:
Veh1_State[v][0] = 1
else:
Veh1_State[v][0] = 0
action = agent.act(state)
print(f"action {action} episode {e} slot_number {slot_number}")
reward, next_state = env.step(action, slot_number)
episode_reward += reward
next_state = np.reshape(next_state, [1, 100])
agent.memorize(state, action, reward, next_state, True)
state = next_state
# print(f"agent.memory {len(agent.memory)} next state size {sys.getsizeof(state)}")
if len(agent.memory) > batch_size:
agent.replay(batch_size, slot_number)
# print(f"reward {reward} episode_reward {episode_reward}")
writer.add_scalar("Episode Reward/episode", episode_reward, e)
print(f"writer size {sys.getsizeof(writer)}")
I have tried printing memory of all objects using sys.getsizeof(object) that I could have imagined could use max memory but can't find any object using that much memory. My data in each iteration is an array of [1, 100]. So, 100*500=50000 arrays in all those loops. I have tried running the code in Google Cloud Instances with a 64GB RAM, and it ended up filling around the whole RAM in HEAP allocation. I have 16GB RAM in my laptop and a 32GB swap; all that space is also allocated to HEAP when running this code.
I am unable to get where that memory lead occurs or which object is causing that much memory allocation. I am using PyCharm on my laptop to run the code. Any hints to get the memory leak or any other problem causing the memory allocation issue?
Heap does not go beyond 8GB when I look at its VM size, as shown in the above picture. But the usage keeps increasing, as shown in the below picture.
