.net-core middleware return blank result

Im making a website with an API, the API needs validation so the user only gets his own data. I have written the following middleware to validate the login.

public class ApiAuthenticationMiddleware
{
    private readonly RequestDelegate _next;
    private readonly UserManager<ApplicationUser> _userManager;
    private readonly SignInManager<ApplicationUser> _signInManager;

    public ApiAuthenticationMiddleware(RequestDelegate next,
        SignInManager<ApplicationUser> signInManager,
        UserManager<ApplicationUser> usermanager)
    {
        _next = next;
        _signInManager = signInManager;
        _userManager = usermanager;
    }

    public async  Task Invoke(HttpContext context)
    {
        if (!context.Request.Query.ContainsKey("password") || !context.Request.Query.ContainsKey("email"))
        {
            context.Response.StatusCode = 401; //UnAuthorized
            await context.Response.WriteAsync("Invalid User Key");
            return;
        }
        var email = context.Request.Query["email"];
        var password = context.Request.Query["password"];

        var result = await _signInManager.PasswordSignInAsync(email, password, false, lockoutOnFailure: false);
        if (result.Succeeded)
        {
            await _next.Invoke(context);
        }
        else if (//some more checks)
            context.Response.StatusCode = 401; //UnAuthorized
            await context.Response.WriteAsync("Invalid User Key");
            return;
        }
    }
}

What I want is that a blank page or an error message like "Invalid User Key" is shown if the user has no valid login. However what happens at the moment is the home page is returned (because my middleware is called before usemvc and the return statement skips the controller request done by usemvc).

My relevant code in the configure method in startup.cs

    public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory)
    {

        app.UseIdentity();
        app.UseWhen(x => (x.Request.Path.StartsWithSegments("/api", StringComparison.OrdinalIgnoreCase)),
            builder =>
            {
                builder.UseMiddleware<ApiAuthenticationMiddleware>();
            });
        app.UseStaticFiles();
        app.UseMvc(routes =>
        {
            routes.MapRoute(
                name: "default",
                template: "{controller=Home}/{action=Index}/{id?}");
        });
    }

How can I make my middleware return a blank page with status code?

Update

After trying alot of things I found that when I remove:

 context.Response.StatusCode = 401; //UnAuthorized

It works as expected, the user gets the message given in:

await context.Response.WriteAsync("Invalid User Key"); 

Howerver I dont want the user to get a 200 statuscode on login fail but a 401 statuscode. But when using the statuscode line the user is redirected. So how can I send the statuscode without redirection?