.Net Core API CORS, lock down to only one domain (Angular app)

I have a .net core api which I want to accept calls from one domain only. There is an Angular 4 application hosted there.

I am creating a cors policy like this...

    services.AddCors(options =>
        {
            options.AddPolicy("AllowSpecificOrigin",
                builder => builder.WithOrigins("http://www.test.com").AllowAnyHeader());
        });

And am loading it like this...

  app.UseCors("AllowSpecificOrigin");

I would have expected only calls from the Angular app on the domain to be able to hit the api, however if I create a call in Chrome Postman I can still hit the api directly.

What am I doing wrong? I want to lock down all traffic apart from the Angular app.

Here are more details of my startup.cs...

    public Startup(IHostingEnvironment env)
    {
        var builder = new ConfigurationBuilder()
            .SetBasePath(env.ContentRootPath)
            .AddJsonFile("appsettings.json", true, true)
            .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true)
            .AddEnvironmentVariables();


        Configuration = builder.Build();
    }

    public void ConfigureServices(IServiceCollection services)
    {
        services.AddMvc();
        services.AddCors();

        services.Configure<AppSettings>(Configuration);
    }

    public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory)
    {
            app.UseCors(builder => builder.WithOrigins(new[] { "http://test.com" }) 
                .AllowCredentials()); 


            app.UseDeveloperExceptionPage();

            app.UseMvc(routes =>
            {
                routes.MapRoute(
                    "default",
                    "{controller=ping}/{action=get}/{id?}");
            });

            app.UseStaticFiles();

        }