Collide RealityKit 3D model with LiDAR mesh

user_3756317 0
Viewed 740

I spent many days trying to understand and follow examples without success. My goal is to place a virtual AR object to the real world scanned previously with the LiDAR. With the showSceneUnderstanding I can see the realtime mesh created ok that fine. With a tap function I can insert a usdz file, that also fine. Because I have toyModel.physicsBody?.mode = .kinematic and self.arView.installGestures(for: toyRobot) I can move/scale the model. Now want to be able move the model AND collide with the mesh generated by the LiDAR. When I move the model to a scanned wall the mesh it's stopped for example.

Here is my complete code :

import UIKit
import RealityKit
import ARKit

class ViewController: UIViewController, ARSessionDelegate {
    
    @IBOutlet var arView: ARView!
    var tapRecognizer = UITapGestureRecognizer()
    
    override func viewDidLoad() {
        super.viewDidLoad()
        
        self.arView.session.delegate = self
        
        //Scene Understanding options
        self.arView.environment.sceneUnderstanding.options.insert([.physics, .collision, .occlusion])
        
        //Only for dev
        self.arView.debugOptions.insert(.showSceneUnderstanding)
        
        self.tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(placeObject(_:)))
        self.arView.addGestureRecognizer(self.tapRecognizer)
        
    }
    
    @objc func placeObject(_ sender: UITapGestureRecognizer) {
        
        // Perform a ray cast against the mesh (sceneUnderstanding)
        // Note: Ray-cast option ".estimatedPlane" with alignment ".any" also takes the mesh into account.
        let tapLocation = sender.location(in: arView)
        if let result = arView.raycast(from: tapLocation, allowing: .estimatedPlane, alignment: .any).first {
            
            // Load the "Toy robot"
            let toyRobot = try! ModelEntity.loadModel(named: "toy_robot_vintage.usdz")
                
            // Add gestures to the toy (only available is physicsBody mode == kinematic)
            self.arView.installGestures(for: toyRobot)
            
            // Toy Anchor to place the toy on surface
            let toyAnchor = AnchorEntity(world: result.worldTransform)
                toyAnchor.addChild(toyRobot)
            
            // Create a "Physics" model of the toy in order to add physics mode
            guard let toyModel = toyAnchor.children.first as? HasPhysics else {
                return
            }
            
            // Because toyModel is a fresh new model we need to init physics
            toyModel.generateCollisionShapes(recursive: true)
            toyModel.physicsBody = .init()
            
            // Add the physics body mode
            toyModel.physicsBody?.mode = .kinematic
            
            let test = ShapeResource.generateConvex(from: toyRobot.model!.mesh)
            
            toyModel.components[CollisionComponent] = CollisionComponent(shapes: [test], mode: .default, filter: .default)
            
            // Finally add the toy anchor to the scene
            self.arView.scene.addAnchor(toyAnchor)
        }
        
    }
}

Someone knows if it's possible to achieve that ? Many thanks in advance!

realitykit arkit augmented-reality swift
2 Answers

Following the previous discussion with @AndyFedoroff I added convext raycast in order, always, to collide the placed 3d object with the LiDAR created mesh. Here is my full code. I don't know if I'm doing well... In any case still doesn't work.

import UIKit
import RealityKit
import ARKit

class ViewController: UIViewController, ARSessionDelegate {
    
    @IBOutlet var arView: ARView!
    
    var tapRecognizer = UITapGestureRecognizer()
    var panGesture = UIPanGestureRecognizer()
    var panGestureEntity: Entity? = nil
    
    override func viewDidLoad() {
        super.viewDidLoad()
        
        self.arView.session.delegate = self
        
        //Scene Understanding options
        self.arView.environment.sceneUnderstanding.options.insert([.physics, .collision, .occlusion])
        
        //Only for dev
        self.arView.debugOptions.insert(.showSceneUnderstanding)
        
        self.tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(self.placeObject))
        self.arView.addGestureRecognizer(self.tapRecognizer)
        
        self.panGesture = UIPanGestureRecognizer(target: self, action: #selector(self.didPan))
        self.arView.addGestureRecognizer(self.panGesture)
    }
    
    @objc func placeObject(_ sender: UITapGestureRecognizer) {
        
        // Perform a ray cast against the mesh (sceneUnderstanding)
        // Note: Ray-cast option ".estimatedPlane" with alignment ".any" also takes the mesh into account.
        let tapLocation = sender.location(in: arView)
        if let result = self.arView.raycast(from: tapLocation, allowing: .estimatedPlane, alignment: .any).first {

            // Load the "Toy robot"
            let toyRobot = try! ModelEntity.loadModel(named: "toy_robot_vintage.usdz")

            // Add gestures to the toy (only available is physicsBody mode == kinematic)
            self.arView.installGestures(for: toyRobot)

            // Toy Anchor to place the toy on surface
            let toyAnchor = AnchorEntity(world: result.worldTransform)
                toyAnchor.addChild(toyRobot)

            // Create a "Physics" model of the toy in order to add physics mode
            guard let model = toyAnchor.children.first as? (Entity & HasPhysics)
            else { return }

            model.generateCollisionShapes(recursive: true)
            model.physicsBody = PhysicsBodyComponent(shapes: [.generateBox(size: .one)],
                                                       mass: 1.0,
                                                   material: .default,
                                                   mode: .kinematic)

            // Finally add the toy anchor to the scene
            self.arView.scene.addAnchor(toyAnchor)
            
        }
    
    }
    
    @objc public func didPan(_ sender: UIPanGestureRecognizer) {
        print(sender.state)
        let point = sender.location(in: self.arView)
        switch sender.state {
        case .began:
            if let entity = self.arView.hitTest(point).first?.entity {
                self.panGestureEntity = entity
            } else {
                self.panGestureEntity = nil
            }
        case .changed:
            if let entity = self.panGestureEntity {
                if let raycast = arView
                    .scene
                    .raycast(origin: .zero, direction: .one, length: 1, query: .all, mask: .all, relativeTo: entity)
                    .first
                {
                    print("hit", raycast.entity, raycast.distance)
                }

          }
        case .ended:
            self.panGestureEntity = nil
        default: break;
        }

    }
    
    
}
user_37563170

Try downcasting to Entity & HasPhysics:

guard let model = anchor.children.first as? (Entity & HasPhysics) 
else { return }

model.generateCollisionShapes(recursive: true)
model.physicsBody = PhysicsBodyComponent(shapes: [.generateBox(size: .one)],
                                           mass: 1.0, 
                                       material: .default, 
                                           mode: .kinematic)

Official documentation says:

For non-model entities, generateCollisionShapes(recursive:) method has no effect. Nevertheless, the method is defined for all entities so that you can call it on any entity, and have the calculation propagate recursively to all that entity’s descendants.

user_65995900
Related