Jasmine testing Angular app with different browsers

I want run my tests locally with Chrome as browser and on the build server with phantomjs. How can I configure that in the karma.conf.js file

module.exports = function (config) {
  config.set({
    basePath: '',
    frameworks: ['jasmine', '@angular/cli'],
    plugins: [
      require('karma-jasmine'),
      require('karma-chrome-launcher'),
      require('karma-phantomjs-launcher'),
      require('karma-jasmine-html-reporter'),
      require('karma-coverage-istanbul-reporter'),
      require('karma-trx-reporter'),
      require('@angular/cli/plugins/karma')
    ],
    client:{
      clearContext: false // leave Jasmine Spec Runner output visible in browser
    },
    files: [
      { pattern: './<%= sourceDir %>/test.ts', watched: false },
    ],
    preprocessors: {
      './<%= sourceDir %>/test.ts': ['@angular/cli']
    },
    mime: {
      'text/x-typescript': ['ts','tsx']
    },
    coverageIstanbulReporter: {
      reports: [ 'html', 'lcovonly' ],
      fixWebpackSourcePaths: true
    },
    angularCli: {
      environment: 'dev'
    },
    reporters: config.angularCli && config.angularCli.codeCoverage
      ? ['progress', 'coverage-istanbul', 'trx']
      : ['progress', 'kjhtml', 'trx'],
    trxReporter: {
      outputFile: 'test-results.trx',
      shortTestName: false
    },
    port: 9876,
    colors: true,
    logLevel: config.LOG_INFO,
    autoWatch: true,
    browsers: ['Chrome'],
    singleRun: false
  });
};

on my build server I use this config:

browsers: ['PhantomJs'],
singleRun: true

I run the tests on my build server by running npm test, which launches ng test. Is there a way to configure that directly on this config file or I should use different karma.conf.js files? And how I should use this. Or should I use some arguments on ng test?