Deferred Screenspace Decals in Metal

Viewed 154

Trying to create deferred screenspace decals rendering in Metal by following this article. Though can't seem to figure it out...

These are bounds of the decal...

decal bounds

Actual result...

result

Potential issue

So apparently it doesn't think that the decal is intersecting the mesh, I'm sampling the depth value correctly, but then when calculating the actual position the the pixel in 3D space something doesn't add up.

Code

vertex VertexOut vertex_decal(
    const VertexIn in [[ stage_in ]],
    constant DecalVertexUniforms &uniforms [[ buffer(2) ]]
) {
    VertexOut out;

    out.position = uniforms.projectionMatrix * uniforms.viewMatrix * uniforms.modelMatrix * in.position;
    out.viewPosition = (uniforms.viewMatrix * uniforms.modelMatrix * in.position).xyz;
    out.normal = uniforms.normalMatrix * in.normal;
    out.uv = in.uv;
    
    return out;
}

fragment float4 fragment_decal(
    const VertexOut in [[ stage_in ]],
    constant DecalFragmentUniforms &uniforms [[ buffer(3) ]],
    depth2d<float, access::sample> depthTexture [[ texture(0) ]]
) {
    constexpr sampler textureSampler (mag_filter::nearest, min_filter::nearest);

    float2 resolution = float2(
        depthTexture.get_width(),
        depthTexture.get_height()
    );
    
    float2 textureCoordinate = in.position.xy / resolution;
    float depth = depthTexture.sample(textureSampler, textureCoordinate);
    
    float3 viewRay = in.viewPosition * (uniforms.farClipPlane / in.viewPosition.z);

    float3 viewPosition = viewRay * depth;
    float3 worldPositon = (uniforms.inverseViewMatrix * float4(viewPosition, 1)).xyz;
    float3 objectPositon = (uniforms.inverseModelMatrix * float4(worldPositon, 1)).xyz;
    
    float distX = 0.5 - abs(objectPositon.x);
    float distY = 0.5 - abs(objectPositon.y);
    float distZ = 0.5 - abs(objectPositon.z);

    if(distX > 0 && distY > 0 && distZ > 0) {
        return float4(1, 0, 0, 0.5);
    } else {
        discard_fragment();
    }
}

EDIT:

Made a bit of a progress, now it at least renders something, it clips the decal box correctly once its outside of some mesh, but the parts on the mesh are still not completely correct.. to be exact it also renders sides of the box that are overlapping with the mesh under the decal (you can see it on the image below as the red there is a bit darker)

depth issue

And to add more details, the depthTexture is passed from previous "pass" so it only contains the icosphere on it, and the decal cube shader doesn't write to the depthTexture, just reads from it.

and depth stencil is defined as...

let stencilDescriptor = MTLDepthStencilDescriptor()
stencilDescriptor.depthCompareFunction = .less
stencilDescriptor.isDepthWriteEnabled = false

and render pipeline is defined as...

let renderPipelineDescriptor = MTLRenderPipelineDescriptor()
renderPipelineDescriptor.vertexDescriptor = vertexDescriptor

renderPipelineDescriptor.vertexFunction = vertexLibrary.makeFunction(name: "vertex_decal")
renderPipelineDescriptor.fragmentFunction = fragmentLibrary.makeFunction(name: "fragment_decal")

if let colorAttachment = renderPipelineDescriptor.colorAttachments[0] {
    colorAttachment.pixelFormat = .bgra8Unorm
    colorAttachment.isBlendingEnabled = true
    colorAttachment.rgbBlendOperation = .add
    colorAttachment.sourceRGBBlendFactor = .sourceAlpha
    colorAttachment.destinationRGBBlendFactor = .oneMinusSourceAlpha
}

renderPipelineDescriptor.colorAttachments[1].pixelFormat = .bgra8Unorm
renderPipelineDescriptor.depthAttachmentPixelFormat = .depth32Float

so the current issue is that it discards only pixels that are out of the mesh that its being projected on, instead of all pixels that are "above" the surface of the icosphere

New Shader Code

fragment float4 fragment_decal(
    const VertexOut in [[ stage_in ]],
    constant DecalFragmentUniforms &uniforms [[ buffer(3) ]],
    depth2d<float, access::sample> depthTexture [[ texture(0) ]]
) {
    constexpr sampler textureSampler (mag_filter::nearest, min_filter::nearest);

    float2 resolution = float2(
        depthTexture.get_width(),
        depthTexture.get_height()
    );
        
    float2 textureCoordinate = in.position.xy / resolution;
    float depth = depthTexture.sample(textureSampler, textureCoordinate);
    
    float3 screenPosition = float3(textureCoordinate * 2 - 1, depth);
    float4 viewPosition = uniforms.inverseProjectionMatrix * float4(screenPosition, 1);
    float4 worldPosition = uniforms.inverseViewMatrix * viewPosition;
    float3 objectPosition = (uniforms.inverseModelMatrix * worldPosition).xyz;
    
    if(abs(worldPosition.x) > 0.5 || abs(worldPosition.y) > 0.5 || abs(worldPosition.z) > 0.5) {
        discard_fragment();
    } else {
        return float4(1, 0, 0, 0.5);
    }
}
1 Answers

Finally managed to get it to work properly, so the final shader code is...

the issues that the latest shader had were...

  • Flipped Y axis on screenPosition
  • Not converting the objectPosition to NDC space (localPosition)
fragment float4 fragment_decal(
    const VertexOut in [[ stage_in ]],
    constant DecalFragmentUniforms &uniforms [[ buffer(3) ]],
    depth2d<float, access::sample> depthTexture [[ texture(0) ]],
    texture2d<float, access::sample> colorTexture [[ texture(1) ]]
) {
    constexpr sampler depthSampler (mag_filter::linear, min_filter::linear);

    float2 resolution = float2(
        depthTexture.get_width(),
        depthTexture.get_height()
    );
        
    float2 depthCoordinate = in.position.xy / resolution;
    float depth = depthTexture.sample(depthSampler, depthCoordinate);
    
    float3 screenPosition = float3((depthCoordinate.x * 2 - 1), -(depthCoordinate.y * 2 - 1), depth);
    float4 viewPosition = uniforms.inverseProjectionMatrix * float4(screenPosition, 1);
    float4 worldPosition = uniforms.inverseViewMatrix * viewPosition;
    float4 objectPosition = uniforms.inverseModelMatrix * worldPosition;
    float3 localPosition = objectPosition.xyz / objectPosition.w;
    
    if(abs(localPosition.x) > 0.5 || abs(localPosition.y) > 0.5 || abs(localPosition.z) > 0.5) {
        discard_fragment();
    } else {
        float2 textureCoordinate = localPosition.xy + 0.5;
        float4 color = colorTexture.sample(depthSampler, textureCoordinate);
        
        return float4(color.rgb, 1);
    }
}

The final results look like this (red are pixels that are kept, blue pixels are discarded)...

final result

Related