UseServiceProviderFactory doesn't affect HttpContext.RequestServices

I am trying to override service provider factory by using UseServiceProviderFactory method. The problem is, while it seems to affect app.Services, it does not affect HttpContext.RequestServices, and also does not work with Controller. Here is my code for proof-of-concept:

using System.Diagnostics;
using System.Text.Json;

var builder = WebApplication.CreateBuilder(args);
builder.Host.UseServiceProviderFactory(new MyFactory());
var app = builder.Build();

// works fine here
var i = app.Services.GetService<MyInstance>();
Debug.Assert(i != null);
Console.WriteLine(i.X);

app.MapGet("/", async c =>
{
    // i is always null here
    var i = c.RequestServices.GetService<MyInstance>();
    await c.Response.WriteAsJsonAsync(i);
});

app.Run();

// classes 

class ContainerBuilder
{
    public ContainerBuilder(IServiceCollection serviceDescriptors)
    {
        ServiceDescriptors = serviceDescriptors;
    }

    public IServiceCollection ServiceDescriptors { get; }
}

class MyInstance { public int X => 42; }

class MyServiceScopeFactory : IServiceScopeFactory
{
    public IServiceScope CreateScope()
    {
        // never got called
        throw new NotImplementedException();
    }
}

class MyServiceProvider : IServiceProvider
{
    private readonly ServiceProvider _services;

    public MyServiceProvider(ServiceProvider services) => _services = services;

    public object? GetService(Type serviceType)
    {        
        if (serviceType == typeof(MyInstance)) 
            return new MyInstance();
        else if (serviceType == typeof(IServiceScopeFactory)) 
            // this line never got hit
            return new MyServiceScopeFactory();
        else return _services.GetService(serviceType);
    }
}

class MyFactory : IServiceProviderFactory<ContainerBuilder>
{
    public ContainerBuilder CreateBuilder(IServiceCollection services)
    {
        return new ContainerBuilder(services);
    }

    public IServiceProvider CreateServiceProvider(ContainerBuilder containerBuilder)
    {
        return new MyServiceProvider(containerBuilder.ServiceDescriptors.BuildServiceProvider());
    }
}

I have checked and compared with AutoFac's implementation but I cannot see what I am missing.