How can I allow for the player to detach from a wall after initiating climbing?

I am admittedly trying to hack and slash some code together for a simple free climbing system in unity. It's my first intermediate project and has turned out to be a bit more complicated than I thought. I have a character controller than when jumping towards a wall, will disable the character controller script and switch to a climbing controller script.

I am trying to allow for the player to press space to jump off of the wall, and back to the normal character controller script, but whenever I implement any sort of jumping behavior, the entire thing falls apart. Can anyone give me a push in the right direction or a simple clue as to how to allow for the character to detach cleanly from the wall?

Here is the code:

    climbing

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class Climbing : MonoBehaviour
{
    [SerializeField] private float speed = 5f;

    Rigidbody rb;

    void Start()
    {
        rb = GetComponent<Rigidbody>();
        rb.useGravity = false;
    }

    private void Update()
    {

    }

    void FixedUpdate()
    {
        float h = Input.GetAxis("Horizontal");
        float v = Input.GetAxis("Vertical");
        Vector2 input = SquareToCircle(new Vector2(h, v));
        {
            // Check walls in a cross pattern
            Vector3 offset = transform.TransformDirection(Vector2.one * 0.5f);
            Vector3 checkDirection = Vector3.zero;
            int k = 0;
            for (int i = 0; i < 4; i++)
            {
                RaycastHit checkHit;
                if (Physics.Raycast(transform.position + offset,
                                    transform.forward,
                                    out checkHit))
                {
                    checkDirection += checkHit.normal;
                    k++;
                }
                // Rotate Offset by 90 degrees
                offset = Quaternion.AngleAxis(90f, transform.forward) * offset;
            }
            checkDirection /= k;

            // Check wall directly in front
             RaycastHit hit;
            if (Physics.Raycast(transform.position, // Position
                                transform.forward, // Direction
                                out hit)) // Hit Data
            {
                transform.forward = -hit.normal;
                rb.position = Vector3.Lerp(rb.position,
                                            hit.point + hit.normal * 0.51f,
                                            10f * Time.fixedDeltaTime);
            }

            rb.velocity = transform.TransformDirection(input) * speed;
        }

        Vector2 SquareToCircle(Vector2 input)
        {

            return (input.sqrMagnitude >= 1f) ? input.normalized : input;
        }
    }

}