OpenCV waitKey() and waitKeyEx() fail to detect arrow keys

I've tried using waitKey() or waitKeyEx() functions to detect my arrow keys but it fails returning "-1". I know they return different values, and I changed them accordingly.

I've built a program with a lot of key bindings, ranging from "Alt", "Shift", "Spacebar", "Enter", "Escape", normal letters and number, and finally the left and right arrow key. Basically when I'm running the program from the beginning, it all works great, but if I pause it (on key press begins a while loop doing nothing), and do some other stuff in another window (e.g. browser), when I resume the program, all keys work as expected, minus the left and right arrow key.

Does anyone know what could be the issue? Only the arrow keys return the "-1"...

EDIT (added example)

(running on Ubuntu 18.04.4 LTS)

import numpy as np
import cv2

# generate random video
video = np.zeros((40,512,512))
for idx, aux in enumerate(video):
    img_ = np.random.randint(255, size=(512,512))
    img = np.asmatrix(img_)
    video[idx,::] = img

# continuous time flag
continuous = True
# loop flag
to_do = True
idx = 0
while to_do:
    img = np.copy(video[idx, ::].astype('uint8'))

    # draw some text
    cv2.putText(img,
                'Something',
                (50, 50),
                cv2.FONT_HERSHEY_DUPLEX,
                0.5,
                (255, 0, 0),
                1)

    cv2.imshow("Window", img)
    # checks if it's by frame or not
    if continuous:
        idx += 1
    else:
        idx = idx

    # check for key press
    k = cv2.waitKeyEx(50)
    print(k)
    # 'SHIFT' -> by frame/continuos
    if k == 65506:
        if continuous:
            continuous = False
        else:
            continuous = True
    # right arrow key -> next frame
    elif k == 65363:
        idx += 1
    # left arrow key -> previous frame
    elif k == 65361:
        idx -= 1
    # 'Alt' -> Pause
    elif k == 65513:
        paused = True
        while paused:
            ka = cv2.waitKeyEx(100)
            if ka == 65513:
                paused = False
        k = None
    # '\' -> exit now
    elif k == 92:
        to_do = False

    if idx == len(video):
        idx = 0
    if idx < 0:
        idx = len(video)-1

cv2.destroyAllWindows()