How to config AddAuthorization on .net core in order to allow roles for multiple token providers?

I am building an API that needs to be authenticated using its own token generated by OpenIddict or ADFS and includer roles on its controller [Authorize(Roles = "A1, A2")]. However if add any option which is not the default the roles does not work:

My startup.cs ConfigureServices method

services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
.AddJwtBearer(options =>
{
    options.Authority = authenticationOptions.Issuer;
    options.Audience = authenticationOptions.Audience;
    options.RequireHttpsMetadata = !hostingEnvironment.IsDevelopment();
    options.IncludeErrorDetails = true;
    options.SaveToken = true;

    options.TokenValidationParameters = new TokenValidationParameters
    {
        ValidateLifetime = true,
        ValidateIssuer = true,
        ValidIssuer = authenticationOptions.Issuer,
        ValidateAudience = true,
        ValidAudience = authenticationOptions.Audience,
        ValidateIssuerSigningKey = true,
        IssuerSigningKey = new X509SecurityKey(LoadCertificate(services))
    };
})
.AddJwtBearer("ADFS", options =>
{
    options.Authority = appSettings.AdfsAuthority;
    options.Audience = appSettings.AdfsAudience;
    options.IncludeErrorDetails = true;
    options.MetadataAddress = appSettings.AdfsMetadataAddress;

    options.TokenValidationParameters = new TokenValidationParameters
    {
        ValidateLifetime = true,
        ValidateIssuer = true,
        ValidIssuer = appSettings.AdfsIssuer,
        ValidateAudience = true,
        ValidAudience = appSettings.AdfsAudience,
        ValidateIssuerSigningKey = true
    };
});

services.AddAuthorization(options =>
{
    options.AddPolicy("A2", policy => policy.RequireRole("A2"));
    //options.AddPolicy("A1", policy => policy.RequireRole("A1"));

    var policies = new AuthorizationPolicyBuilder(JwtBearerDefaults.AuthenticationScheme, "ADFS")
        .RequireAuthenticatedUser()
        .RequireRole("A1")
        .Build();

    options.DefaultPolicy = policies;
});

If I use just one AddJwtBearer provider the roles policy works as below:

services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
.AddJwtBearer(options =>
{
    options.Authority = authenticationOptions.Issuer;
    options.Audience = authenticationOptions.Audience;
    options.RequireHttpsMetadata = !hostingEnvironment.IsDevelopment();
    options.IncludeErrorDetails = true;
    options.SaveToken = true;

    options.TokenValidationParameters = new TokenValidationParameters
    {
        ValidateLifetime = true,
        ValidateIssuer = true,
        ValidIssuer = authenticationOptions.Issuer,
        ValidateAudience = true,
        ValidAudience = authenticationOptions.Audience,
        ValidateIssuerSigningKey = true,
        IssuerSigningKey = new X509SecurityKey(LoadCertificate(services))
    };
});

services.AddAuthorization(options =>
{
    options.AddPolicy("A2", policy => policy.RequireRole("A2"));
});

Or

services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
.AddJwtBearer(options =>
{
    options.Authority = appSettings.AdfsAuthority;
    options.Audience = appSettings.AdfsAudience;
    options.IncludeErrorDetails = true;
    options.MetadataAddress = appSettings.AdfsMetadataAddress;

    options.TokenValidationParameters = new TokenValidationParameters
    {
        ValidateLifetime = true,
        ValidateIssuer = true,
        ValidIssuer = appSettings.AdfsIssuer,
        ValidateAudience = true,
        ValidAudience = appSettings.AdfsAudience,
        ValidateIssuerSigningKey = true
    };
});

services.AddAuthorization(options =>
{
    options.AddPolicy("A1", policy => policy.RequireRole("A1"));
});

Could anyone help with that? I guess the problem is on AddAuthorization because somehow the RequireRole config is always applied to default schema it is never applied for the custom one "ADFS" in this case.