Unity XR - Controller based on VR Hands, Spins in random directions, should rotate the game object slowly in the direction of VR hands rotation

I'm running into an issue with my Space Ship controller system, I'm using Unity XR to control the space ship based on the rotation of the XR Hands, however, when attached the Joystick in game goes crazy, as well as the Space Ship. I'm looking for a controller which gradually shifts the direction of the space ship, and then Joystick to mimic the input of the XR controller.

Example Video of What Goes Wrong: Youtube - Issue with Game

Below is the script I have attached to the joystick collider:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System;
using Unity.XR.CoreUtils;
using UnityEngine.XR;
using UnityEngine.XR.Interaction.Toolkit;

public class joystickControl : MonoBehaviour
{
    public Transform stick;
    public GameObject joyStick;
    [SerializeField]
    private Transform spaceShip;
    [SerializeField]
    private float forwardTilt = 0;
    [SerializeField]
    private float sideTilt = 0;
    [SerializeField]
    private float damper = 1000;
    private float roll = 0;
    private float pitch = 0;
    private float yaw = 0;
    public float rollCorrected;
    public float pitchCorrected;
    public float yawCorrected;
    private InputDevice hand;
    public XRNode inputSource;
    public GameObject handVector;
    private bool controlling;
    private bool triggerValue;

    void Start(){
    }

    void Update(){

        //This needs to have a deadzone and a limit, otherwise it gives too much control, and makes the ship uncontrolable.

        if(controlling){
            Debug.Log("Controlling");
            //hand.TryGetFeatureValue(CommonUsages.deviceAngularVelocity, out Vector3 handRot);
            Vector3 handRot = handVector.transform.eulerAngles;
            Debug.Log(handRot);

            pitch = handRot.x;
            if(pitch < 355 && pitch > 290){
                pitchCorrected =  Math.Abs(pitch - 360)/damper;
            }
            else if (pitch > 5 && pitch < 74){
                pitchCorrected =  pitch/damper;
            }
            roll = handRot.y;
            if(roll < 355 && roll > 290){
                rollCorrected =  Math.Abs(roll - 360)/damper;
            }
            else if (roll > 5 && roll > 290){
                rollCorrected  =  roll/damper;
            }
            yaw = handRot.z;
            if (yaw < 355 && yaw > 290){
                yawCorrected =  Math.Abs(yaw-360)/damper;
            }
            else if (yaw > 5 && yaw > 290){
                yawCorrected = yaw/damper;
            }
            Debug.Log(rollCorrected + " " + pitchCorrected + " " + yawCorrected);
            joyStick.transform.Rotate(pitchCorrected * damper, rollCorrected * damper, yawCorrected * damper);
            spaceShip.Rotate(0f, 0f, rollCorrected);
            spaceShip.Rotate(pitchCorrected, 0f, 0f);
            spaceShip.Rotate(0f, yawCorrected, 0f);
        }
    }
    void OnTriggerEnter(Collider other){
        if (other.CompareTag("PlayerHand")){
            InputDevice hand = InputDevices.GetDeviceAtXRNode(inputSource);
        hand.TryGetFeatureValue(CommonUsages.gripButton, out bool triggerValue);
            Debug.Log("JoyStickControl - Player Hand Collision");
            Debug.Log("JoyStickControl - triggerValue - " + triggerValue);
            if(triggerValue){
                controlling = true;
                Debug.Log("JoyStickControl - True");
            }
            else{
                Debug.Log("JoyStickControl - False");
            }
        }
    }
}

Thanks in advance for your help!