Custom IServiceProviderFactory in ASP.NET Core

I wrote a custom IServiceProviderFactory and installed it in Program.cs of a new app like this:

    public static IHostBuilder CreateHostBuilder(string[] args) =>
        Host.CreateDefaultBuilder(args)
            .UseServiceProviderFactory(new PropertyInjectingContainerFactory())
            .ConfigureWebHostDefaults(webBuilder =>
            {
                webBuilder.UseStartup<Startup>();
            });

It does lead to the correct configure function in Startup.cs getting called:

   public void ConfigureContainer(PropertyInjectingContainerFactory.Builder builder)
   {
        builder.AddInjectAttribute<InjectDependencyAttribute>();
   }

However, my created container only ever resolves two services: IConfiguration and IHost.

Everything else is resolved by the default container apparantly (for instance a service like ILogger<T> on a controller). What do I do wrong?

Here's the code for my custom factory - and please understand that I probably should be using an existing third-party container, but I also want to understand how this all fits together.

public class PropertyInjectingContainerFactory : IServiceProviderFactory<PropertyInjectingContainerFactory.Builder>
{
    public Builder CreateBuilder(IServiceCollection services)
    {
        return new Builder(services);
    }

    public IServiceProvider CreateServiceProvider(Builder containerBuilder)
    {
        return containerBuilder.CreateServiceProvider();
    }

    public class Builder
    {
        internal readonly IServiceCollection services;

        internal List<Type> attributeTypes = new List<Type>();

        public Builder(IServiceCollection services)
        {
            this.services = services;
        }

        public Builder AddInjectAttribute<A>()
            where A : Attribute
        {
            attributeTypes.Add(typeof(A));

            return this;
        }

        public IServiceProvider CreateServiceProvider()
            => new PropertyInjectingServiceProvider(services.BuildServiceProvider(), attributeTypes.ToArray());
    }

    class PropertyInjectingServiceProvider : IServiceProvider
    {
        private readonly IServiceProvider services;
        private readonly Type[] injectAttributes;

        public PropertyInjectingServiceProvider(IServiceProvider services, Type[] injectAttributes)
        {
            this.services = services;
            this.injectAttributes = injectAttributes;
        }

        // This function is only called for `IConfiguration` and `IHost` - why?
        public object GetService(Type serviceType)
        {
            var service = services.GetService(serviceType);

            InjectProperties(service);

            return service;
        }

        private void InjectProperties(Object target)
        {
            var type = target.GetType();

            var candidateProperties = type.GetProperties(System.Reflection.BindingFlags.Public);

            var props = from p in candidateProperties
                        where injectAttributes.Any(a => p.GetCustomAttributes(a, true).Any())
                        select p;

            foreach (var prop in props)
            {
                prop.SetValue(target, services.GetService(prop.PropertyType));
            }
        }
    }
}