redeclaration of import Buffer

I'm getting the error redeclaration of import Buffer when trying to polyfill Buffer in sveltekit.

Here is my svelte.config.js file:

import adapter from '@sveltejs/adapter-static';
import preprocess from 'svelte-preprocess';
import path from 'path';
import { NodeGlobalsPolyfillPlugin } from '@esbuild-plugins/node-globals-polyfill';
import { NodeModulesPolyfillPlugin } from '@esbuild-plugins/node-modules-polyfill';
import dotenv from 'dotenv-flow';
import rollupNodePolyFill from 'rollup-plugin-node-polyfills';

dotenv.config();
const config = {
    preprocess: preprocess(),
    kit: {
        adapter: adapter({
            // default options are shown
            pages: 'build',
            assets: 'build',
            fallback: 'index.html',
            precompress: false
        }),
        vite: {
            optimizeDeps: {
                esbuildOptions: {
                    // Node.js global to browser globalThis
                    define: {
                        global: 'globalThis'
                    },
                    // Enable esbuild polyfill plugins
                    plugins: [
                        NodeGlobalsPolyfillPlugin({
                            buffer: true,
                            global: true,
                            process: true,
                            url: true,
                            assert: true,
                            crypto: true,
                            http: true,
                            https: true,
                            os: true,
                            stream: true,
                            util: true
                        }),
                        NodeModulesPolyfillPlugin()
                    ]
                }
            },
            resolve: {
                alias: {
                    $components: path.resolve('./src/components'),
                    $stores: path.resolve('./src/stores'),
                    $api: path.resolve('./src/api'),
                    $node: path.resolve('./node_modules'),
                    '@toruslabs/openlogin': path.resolve(
                        './node_modules/@toruslabs/openlogin/dist/openlogin.umd.min.js'
                    ),
                    // polyfills
                    util: 'rollup-plugin-node-polyfills/polyfills/util',
                    sys: 'util',
                    // buffer: 'rollup-plugin-node-polyfills/polyfills/buffer-es6',
                    events: 'rollup-plugin-node-polyfills/polyfills/events',
                    stream: 'rollup-plugin-node-polyfills/polyfills/stream',
                    path: 'rollup-plugin-node-polyfills/polyfills/path',
                    querystring: 'rollup-plugin-node-polyfills/polyfills/qs',
                    punycode: 'rollup-plugin-node-polyfills/polyfills/punycode',
                    url: 'rollup-plugin-node-polyfills/polyfills/url',
                    string_decoder: 'rollup-plugin-node-polyfills/polyfills/string-decoder',
                    http: 'rollup-plugin-node-polyfills/polyfills/http',
                    https: 'rollup-plugin-node-polyfills/polyfills/http',
                    os: 'rollup-plugin-node-polyfills/polyfills/os',
                    assert: 'rollup-plugin-node-polyfills/polyfills/assert',
                    constants: 'rollup-plugin-node-polyfills/polyfills/constants',
                    _stream_duplex: 'rollup-plugin-node-polyfills/polyfills/readable-stream/duplex',
                    _stream_passthrough: 'rollup-plugin-node-polyfills/polyfills/readable-stream/passthrough',
                    _stream_readable: 'rollup-plugin-node-polyfills/polyfills/readable-stream/readable',
                    _stream_writable: 'rollup-plugin-node-polyfills/polyfills/readable-stream/writable',
                    _stream_transform: 'rollup-plugin-node-polyfills/polyfills/readable-stream/transform',
                    timers: 'rollup-plugin-node-polyfills/polyfills/timers',
                    console: 'rollup-plugin-node-polyfills/polyfills/console',
                    vm: 'rollup-plugin-node-polyfills/polyfills/vm',
                    zlib: 'rollup-plugin-node-polyfills/polyfills/zlib',
                    tty: 'rollup-plugin-node-polyfills/polyfills/tty',
                    domain: 'rollup-plugin-node-polyfills/polyfills/domain'
                }
            },
            build: {
                minify: false,
                rollupOptions: {
                    plugins: [
                        // Enable rollup polyfills plugin
                        // used during production bundling
                        rollupNodePolyFill({
                            // buffer: false
                        })
                    ]
                }
            }
        }
    }
};

export default config;

So sveltekit uses esbuild for dev and rollup for production builds...so we need two modules.

I almost got my code working but the the polyfilled imports seem to be importing Buffer twice.