How to throwing 404 on bad .NET Core API route?

I have a .NET Core web app which has an API. I've defined an Middleware class based on this answer like so:

public class ErrorHandlingMiddleware
{
    private readonly RequestDelegate next;
    private readonly ILogger logger;

    public ErrorHandlingMiddleware(RequestDelegate next,
        ILoggerFactory loggerFactory)
    {
        this.next = next;
        logger = loggerFactory.CreateLogger<ErrorHandlingMiddleware>();
    }

    public async Task Invoke(HttpContext context)
    {
        try
        {
            await next(context);
        }
        catch (Exception ex)
        {
            logger.LogError(0, ex, "An unhandled exception has occurred: " + ex.StackTrace);
            await HandleExceptionAsync(context, ex);
        }
    }

    private static Task HandleExceptionAsync(HttpContext context, Exception exception)
    {
        var code = HttpStatusCode.InternalServerError;
        var message = exception.Message;
        if (exception is BadRequestException)
        {
            code = HttpStatusCode.BadRequest;
        }
        else if (exception is NotFoundException)
        {
            code = HttpStatusCode.NotFound;
        }
        else if (exception is NotAuthorizedException)
        {
            code = HttpStatusCode.Forbidden;
        }
        else if (exception is NotAuthenticatedException)
        {
            code = HttpStatusCode.Unauthorized;
        }
        else
        {
            message = "An unexpected error occurred.";
        }

        var result = JsonConvert.SerializeObject(new { error = message });
        context.Response.ContentType = "application/json";
        context.Response.StatusCode = (int)code;
        return context.Response.WriteAsync(result);
    }
}

The error handling only handles when an exception is thrown in code. A bad route does not throw an exception. The problem is that if I try to access a non-existent API route - that is, one that follows the API route convention and starts with "/api/adfasdf" - the API returns HTML (or the error page or the home page, I forget).

I've received some suggestions to check the context.Response.StatusCode after await next(context); executes, but it's 200.

How can I configure my web app such that it recognizes a bad API route and returns a 404?

UPDATE Here is where/when I load the middleware in my Startup class:

public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory, IApplicationLifetime appLifetime, IOptions<OidcConfig> oidcConfigOptions)
{
    loggerFactory.AddConsole(Configuration.GetSection("Logging"));
    loggerFactory.AddDebug();

    // Add Serilog to the logging pipeline
    loggerFactory.AddSerilog();

    app.UseMiddleware<ErrorHandlingMiddleware>();

    if (env.IsLocal())
    {
        app.UseWebpackDevMiddleware(new WebpackDevMiddlewareOptions
        {
            HotModuleReplacement = true
        });
    }

    var oidcConfig = oidcConfigOptions.Value;

    // Configure the app to use Jwt Bearer Authentication
    app.UseJwtBearerAuthentication(new JwtBearerOptions
    {
        AutomaticAuthenticate = true,
        AutomaticChallenge = true,
        Authority = oidcConfig.GetAuthority(),
        Audience = oidcConfig.ResourceAppId,
        TokenValidationParameters = new TokenValidationParameters
        {
            RequireExpirationTime = true,
            RequireSignedTokens = true,
            ValidateAudience = true,
            ValidIssuer = oidcConfig.GetIssuer(),
            ValidateIssuer = true,
            ValidateActor = false,
            ValidateLifetime = true,
            ValidateIssuerSigningKey = true
        },
    });

    app.UseSiteIdClaimInjection();

    app.UseStaticFiles();

    app.UseMvc(routes =>
    {
        routes.MapRoute(
            name: "default",
            template: "{controller=Home}/{action=Index}/{id?}");

        routes.MapSpaFallbackRoute(
            name: "spa-fallback",
            defaults: new { controller = "Home", action = "Index" });
    });

    appLifetime.ApplicationStopped.Register(() => this.ApplicationContainer.Dispose());
}