OpenCV - How to push back Mat in a queue?

I am trying to put the frames of a video in a deque. This code does not work. Because the back and front of the queue are both the same as the current frame.

deque<Mat> frameSeq;
int main() {
    Mat frame;
    VideoCapture video("path to video");
    int key = 0;
    while (key != 'q') {
        video >> frame;

        frameSeq.push_back(frame);

        imshow("front", frameSeq.front());
        imshow("back", frameSeq.back());

        key = cvWaitKey(1);
    }
    return 0;
}

But when I resize the frame:

deque<Mat> frameSeq;
int main() {
    Mat frame;
    VideoCapture video("path to video");
    int key = 0;
    while (key != 'q') {
        video >> frame;

        cv::resize(frame, frame, cv::Size(), 1.0 / 2, 1.0 / 2);
        frameSeq.push_back(frame);

        imshow("front", frameSeq.front());
        imshow("back", frameSeq.back());

        key = cvWaitKey(1);
    }
    return 0;
}

It works well. Now frameSeq.back() is the current frame and frameSeq.front() is the initial frame.

How can I make the queue work without having to resize the frame?