Face Detection Using KinectV2 and OpenCV

Recently I have been trying to learn OpenCV and was trying to detect faces using the Haar Classifier.

I was successful in detecting faces when I obtained the video stream from the default webcam,but when I use Kinect instead of the default webcam, It still detects face but the frame rate drops tremendously.

The code that I've Written is

int main() {
    string haar_face = "F:\haarcascade_frontalface_default.xml";
    CascadeClassifier haar_cascade;
    haar_cascade.load(haar_face);
    if (haar_cascade.empty()) {
        return -1;
    }
    vector<Rect_<int>> faces;
    bool optionKinect = false;
    cout << "Choose Option\n1.) Kinect \n>1.) WebCam\n";
    int choice;
    cin >> choice;
    if (choice == 1) {
        optionKinect = true;
    }   
    if (optionKinect) {
    CKinectStreamsMat* kinectStream = new CKinectStreamsMat();
    kinectStream->initSensor();
        while (true) {
            Mat original, gray;
            Mat face;
            Rect face_i;
            //cap >> original;
            original = kinectStream->getColorFrame();
            if (original.data) {
                cvtColor(original, gray, CV_BGR2GRAY);
                haar_cascade.detectMultiScale(gray, faces);
                int size = faces.size();
                for (size_t i = 0; i < size; i++) {
                    face_i = faces[i];
                    face = gray(face_i);
                    rectangle(original, face_i, CV_RGB(0, 255, 0), 1);
                }
                imshow("original", original);
                if(waitKey(20) == 27){
                    break;
                }
            }
        }
    }
    else {
        VideoCapture cap(0);
        while (true) {
            Mat original, gray;
            Mat face;
            Rect face_i;
            cap >> original;
            //original = kinectStream->getColorFrame();
            if (original.data) {
                cvtColor(original, gray, CV_BGR2GRAY);
                haar_cascade.detectMultiScale(gray, faces);
                int size = faces.size();
                for (size_t i = 0; i < size; i++) {
                    face_i = faces[i];
                    face = gray(face_i);
                    rectangle(original, face_i, CV_RGB(0, 255, 0), 1);
                }
                imshow("original", original);
                if(waitKey(20) == 27){
                    break;
                }
            }
        }

    }
}

And this is how I am obtaining the Color frame from Kinect.

cv::Mat CKinectStreamsMat::getColorFrame()
{
    HRESULT hr = E_FAIL;
    IColorFrame* frame = NULL;
    IFrameDescription* frameDesc;
    cv::Mat colorImage;
    hr = _color_reader->AcquireLatestFrame(&frame);
    if (SUCCEEDED(hr)) {
        hr = frame->get_FrameDescription(&frameDesc);
        if (SUCCEEDED(hr)) {
            int frameWidth = 0, frameHeight = 0;
            hr = frameDesc->get_Width(&frameWidth);
            if (SUCCEEDED(hr)) {
                hr = frameDesc->get_Height(&frameHeight);
            }
            if (SUCCEEDED(hr)) {
                const int imgSize = frameWidth*frameHeight * 4 * sizeof(unsigned char); //4 Channels(BGRA)
                colorImage = cv::Mat(1080,1920,CV_8UC4);
                hr = frame->CopyConvertedFrameDataToArray(imgSize, reinterpret_cast<BYTE*>(colorImage.data), ColorImageFormat_Bgra);
            }
        }
        SafeRelease(frameDesc);
        SafeRelease(frame);
    }
    return colorImage;
}

I thought the reason for low performance might be the difference in resolution of the frames provided by the WebCam and Kinect, So I also tried scaling down the frame provided by Kinect down to the size that was even less than the frame size of WebCam. But still the performance was very low.

As this is only what I could think of and now I am out of ideas, So could anyone please tell what could be the reason for this low performance?