Rotate object based on mouse axis

I am rotating an object that has a rigidbody attached and I am using AddTorque() with mouse axis to rotate it in different axis. It works fine in the beginning but if I keep rotating for a while, then the mouse axis interchanges, that is the object rotates Left/Right if I rotate my mouse in Up/Down direction. Why does this happen and how do I avoid it?

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.EventSystems;
using UnityEngine.InputSystem;

public class RotateObject : MonoBehaviour, IBeginDragHandler, IDragHandler, IEndDragHandler {

    [SerializeField] float rotationSpeed = 20f;
    bool dragging = false;
    Rigidbody rb;
    float x;
    float y;

    // Start is called before the first frame update
    void Start()
    {
        rb = GetComponent<Rigidbody>();
    }

     public void OnBeginDrag (PointerEventData eventData) {
         dragging = true;
     }


    public void OnDrag (PointerEventData eventData) {
           x = Mouse.current.delta.x.ReadValue() * rotationSpeed * Time.deltaTime;
           y = -Mouse.current.delta.y.ReadValue() * rotationSpeed * Time.deltaTime;

           rb.AddTorque(Vector3.down*x);
           rb.AddTorque(Vector3.right*y);

     }

     public void OnEndDrag (PointerEventData eventData) {
         dragging = false;
     }
}