Using SwiftUI with multiple UIGestureRecognizers?

I am currently trying to write an app that uses a navigation similar to apps like Affinity, or Procreate, where you use a single touch/drag to draw, retouch and interact, and two-finger gestures to navigate on the canvas.

I am building the app using SwiftUI as the main framework and include UIKit as necessary.

Unfortunately, SwiftUI does not yet allow for complex gestures like the ones described, but UIKit usually does. So I reverted to using UIKit as my gesture recognisers instead of relying on SwiftUI gestures. This however is causing the issue, that only the topmost gesture recogniser will be called. I was hoping for recognition of multiple simultaneous gestures like demonstrated here, but unfortunately, SwiftUI seems to cause issues with the UIViewRepresentables.

Can someone help me figure out a solution to this?

Important: I am doing this with two separate views because in the long run, they will be used on different views. In the example, however, I have them on the same view for demonstration purposes.

Usage:

ZStack {
    DragGestureView { point in
        print("One Finger")
    } dragEndedCallback: {
        print("One Finger Ended")
    }

    TwoFingerNavigationView { point in
        viewStore.send(.dragChanged(point))
        print("Two Fingers")
    } dragEndedCallback: {
        viewStore.send(.dragEnded)
        print("Two Fingers Ended")
    } pinchedCallback: { value in
        viewStore.send(.magnificationChanged(value))
    } pinchEndedCallback: {
        viewStore.send(.magnificationEnded)
    }

    content()
        .position(viewStore.location)
        .scaleEffect(viewStore.scale * viewStore.offsetScale)
}

DragGestureView

public struct DragGestureView: UIViewRepresentable {
    let delegate = GestureRecognizerDelegate()
    var draggedCallback: ((CGPoint) -> Void)
    var dragEndedCallback: (() -> Void)

    public init(draggedCallback: @escaping ((CGPoint) -> Void), dragEndedCallback: @escaping (() -> Void)) {
        self.draggedCallback = draggedCallback
        self.dragEndedCallback = dragEndedCallback
    }

    public class Coordinator: NSObject {
        var draggedCallback: ((CGPoint) -> Void)
        var dragEndedCallback: (() -> Void)

        public init(draggedCallback: @escaping ((CGPoint) -> Void),
             dragEndedCallback: @escaping (() -> Void)) {
            self.draggedCallback = draggedCallback
            self.dragEndedCallback = dragEndedCallback
        }

        @objc func dragged(gesture: UIPanGestureRecognizer) {
            if gesture.state == .ended {
                self.dragEndedCallback()
            } else {
                self.draggedCallback(gesture.location(in: gesture.view))
            }
        }
    }

    class GestureRecognizerDelegate: NSObject, UIGestureRecognizerDelegate {
        func gestureRecognizer(_ gestureRecognizer: UIGestureRecognizer, shouldRecognizeSimultaneouslyWith otherGestureRecognizer: UIGestureRecognizer) -> Bool {
            return true
        }
    }

    public func makeUIView(context: UIViewRepresentableContext<DragGestureView>) -> DragGestureView.UIViewType {
        let view = UIView(frame: .zero)
        let gesture = UIPanGestureRecognizer(target: context.coordinator,
                                             action: #selector(Coordinator.dragged))
        gesture.minimumNumberOfTouches = 1
        gesture.maximumNumberOfTouches = 1
        gesture.delegate = delegate
        view.addGestureRecognizer(gesture)
        return view
    }

    public func makeCoordinator() -> DragGestureView.Coordinator {
        return Coordinator(draggedCallback: self.draggedCallback,
                           dragEndedCallback: self.dragEndedCallback)
    }

    public func updateUIView(_ uiView: UIView,
                      context: UIViewRepresentableContext<DragGestureView>) {
    }
}

TwoFingerNavigationView

struct TwoFingerNavigationView: UIViewRepresentable {
    let delegate = GestureRecognizerDelegate()

    var draggedCallback: ((CGPoint) -> Void)
    var dragEndedCallback: (() -> Void)
    var pinchedCallback: ((CGFloat) -> Void)
    var pinchEndedCallback: (() -> Void)

    class Coordinator: NSObject {
        var draggedCallback: ((CGPoint) -> Void)
        var dragEndedCallback: (() -> Void)
        var pinchedCallback: ((CGFloat) -> Void)
        var pinchEndedCallback: (() -> Void)

        var startingDistance: CGFloat? = nil
        var isMagnifying = false
        var startingMagnification: CGFloat? = nil
        var newMagnification: CGFloat = 1.0

        init(draggedCallback: @escaping ((CGPoint) -> Void),
             dragEndedCallback: @escaping (() -> Void),
             pinchedCallback: @escaping ((CGFloat) -> Void),
             pinchEndedCallback: @escaping (() -> Void)) {
            self.draggedCallback = draggedCallback
            self.dragEndedCallback = dragEndedCallback
            self.pinchedCallback = pinchedCallback
            self.pinchEndedCallback = pinchEndedCallback
        }

        @objc func dragged(gesture: UIPanGestureRecognizer) {
            if gesture.state == .ended {
                self.dragEndedCallback()
                self.pinchEndedCallback()
                startingDistance = nil
                isMagnifying = false
                startingMagnification = nil
                newMagnification = 1.0
            } else {
                self.draggedCallback(gesture.translation(in: gesture.view) / (newMagnification))
            }

            var touchLocations: [CGPoint] = []
            for i in 0..<gesture.numberOfTouches{
                touchLocations.append(gesture.location(ofTouch: i, in: gesture.view))
            }

            if touchLocations.count == 2 {
                let distanceVector = (touchLocations[0] - touchLocations[1])
                let distance = sqrt(distanceVector.x * distanceVector.x + distanceVector.y * distanceVector.y)

                guard startingDistance != nil else { startingDistance = distance; return }
                guard distance - startingDistance! > 30 || distance - startingDistance! < -30 || isMagnifying else { return }
                isMagnifying = true;

                if startingMagnification == nil {
                    startingMagnification = distance / 100
                    pinchedCallback(1)
                } else {
                    let magnification = distance / 100
                    newMagnification = magnification / startingMagnification!
                    pinchedCallback(newMagnification)
                }
            }
        }
    }

    class GestureRecognizerDelegate: NSObject, UIGestureRecognizerDelegate {
        func gestureRecognizer(_ gestureRecognizer: UIGestureRecognizer, shouldRecognizeSimultaneouslyWith otherGestureRecognizer: UIGestureRecognizer) -> Bool {
            return true
        }
    }

    func makeUIView(context: UIViewRepresentableContext<TwoFingerNavigationView>) -> TwoFingerNavigationView.UIViewType {
        let view = UIView(frame: .zero)
        let gesture = UIPanGestureRecognizer(target: context.coordinator,
                                             action: #selector(Coordinator.dragged))
        gesture.minimumNumberOfTouches = 2
        gesture.maximumNumberOfTouches = 2
        gesture.delegate = delegate
        view.addGestureRecognizer(gesture)
        return view
    }

    func makeCoordinator() -> TwoFingerNavigationView.Coordinator {
        return Coordinator(draggedCallback: self.draggedCallback,
                           dragEndedCallback: self.dragEndedCallback,
                           pinchedCallback: self.pinchedCallback,
                           pinchEndedCallback: self.pinchEndedCallback)
    }

    func updateUIView(_ uiView: UIView,
                      context: UIViewRepresentableContext<TwoFingerNavigationView>) {
    }
}