How to merge two arrays in JavaScript and de-duplicate items

Viewed 1823585

I have two JavaScript arrays:

var array1 = ["Vijendra","Singh"];
var array2 = ["Singh", "Shakya"];

I want the output to be:

var array3 = ["Vijendra","Singh","Shakya"];

The output array should have repeated words removed.

How do I merge two arrays in JavaScript so that I get only the unique items from each array in the same order they were inserted into the original arrays?

89 Answers

Using a Set (ECMAScript 2015), it will be as simple as that:

const array1 = ["Vijendra", "Singh"];
const array2 = ["Singh", "Shakya"];
console.log(Array.from(new Set(array1.concat(array2))));

The ES6 offers a single-line solution for merging multiple arrays without duplicates by using destructuring and set.

const array1 = ['a','b','c'];
const array2 = ['c','c','d','e'];
const array3 = [...new Set([...array1,...array2])];
console.log(array3); // ["a", "b", "c", "d", "e"]

I know this question is not about array of objects, but searchers do end up here.

so it's worth adding for future readers a proper ES6 way of merging and then removing duplicates

array of objects:

var arr1 = [ {a: 1}, {a: 2}, {a: 3} ];
var arr2 = [ {a: 1}, {a: 2}, {a: 4} ];

var arr3 = arr1.concat(arr2.filter( ({a}) => !arr1.find(f => f.a == a) ));

// [ {a: 1}, {a: 2}, {a: 3}, {a: 4} ]

EDIT:

The first solution is the fastest only when there is few items. When there is over 400 items, the Set solution becomes the fastest. And when there is 100,000 items, it is a thousand times faster than the first solution.

Considering that performance is important only when there is a lot of items, and that the Set solution is by far the most readable, it should be the right solution in most cases

The perf results below were computed with a small number of items


Based on jsperf, the fastest way (edit: if there is less than 400 items) to merge two arrays in a new one is the following:

for (var i = 0; i < array2.length; i++)
    if (array1.indexOf(array2[i]) === -1)
      array1.push(array2[i]);

This one is 17% slower:

array2.forEach(v => array1.includes(v) ? null : array1.push(v));

This one is 45% slower (edit: when there is less than 100 items. It is a lot faster when there is a lot of items):

var a = [...new Set([...array1 ,...array2])];

And the accepted answers is 55% slower (and much longer to write) (edit: and it is several order of magnitude slower than any of the other methods when there is 100 000 items)

var a = array1.concat(array2);
for (var i = 0; i < a.length; ++i) {
    for (var j = i + 1; j < a.length; ++j) {
        if (a[i] === a[j])
            a.splice(j--, 1);
    }
}

https://jsperf.com/merge-2-arrays-without-duplicate

Performance

Today 2020.10.15 I perform tests on MacOs HighSierra 10.13.6 on Chrome v86, Safari v13.1.2 and Firefox v81 for chosen solutions.

Results

For all browsers

  • solution H is fast/fastest
  • solutions L is fast
  • solution D is fastest on chrome for big arrays
  • solution G is fast on small arrays
  • solution M is slowest for small arrays
  • solutions E are slowest for big arrays

enter image description here

Details

I perform 2 tests cases:

  • for 2 elements arrays - you can run it HERE
  • for 10000 elements arrays - you can run it HERE

on solutions A, B, C, D, E, G, H, J, L, M presented in below snippet

// https://stackoverflow.com/a/10499519/860099
function A(arr1,arr2) {
  return _.union(arr1,arr2)
}

// https://stackoverflow.com/a/53149853/860099
function B(arr1,arr2) {
  return _.unionWith(arr1, arr2, _.isEqual);
}

// https://stackoverflow.com/a/27664971/860099
function C(arr1,arr2) {
  return [...new Set([...arr1,...arr2])]
}

// https://stackoverflow.com/a/48130841/860099
function D(arr1,arr2) {
  return Array.from(new Set(arr1.concat(arr2)))
}

// https://stackoverflow.com/a/23080662/860099
function E(arr1,arr2) {
  return arr1.concat(arr2.filter((item) => arr1.indexOf(item) < 0))
}


// https://stackoverflow.com/a/28631880/860099
function G(arr1,arr2) {
  var hash = {};
  var i;
  
  for (i = 0; i < arr1.length; i++) {
    hash[arr1[i]] = true;
  }
  for (i = 0; i < arr2.length; i++) {
    hash[arr2[i]] = true;
  }
  return Object.keys(hash);
}

// https://stackoverflow.com/a/13847481/860099
function H(a, b){
    var hash = {};
    var ret = [];

    for(var i=0; i < a.length; i++){
        var e = a[i];
        if (!hash[e]){
            hash[e] = true;
            ret.push(e);
        }
    }

    for(var i=0; i < b.length; i++){
        var e = b[i];
        if (!hash[e]){
            hash[e] = true;
            ret.push(e);
        }
    }

    return ret;
}



// https://stackoverflow.com/a/1584377/860099
function J(arr1,arr2) {
  function arrayUnique(array) {
      var a = array.concat();
      for(var i=0; i<a.length; ++i) {
          for(var j=i+1; j<a.length; ++j) {
              if(a[i] === a[j])
                  a.splice(j--, 1);
          }
      }

      return a;
  }

  return arrayUnique(arr1.concat(arr2));
}


// https://stackoverflow.com/a/25120770/860099
function L(array1, array2) {
    const array3 = array1.slice(0);
    let len1 = array1.length;
    let len2 = array2.length;
    const assoc = {};

    while (len1--) {
        assoc[array1[len1]] = null;
    }

    while (len2--) {
        let itm = array2[len2];

        if (assoc[itm] === undefined) { // Eliminate the indexOf call
            array3.push(itm);
            assoc[itm] = null;
        }
    }

    return array3;
}

// https://stackoverflow.com/a/39336712/860099
function M(arr1,arr2) {
  const comp = f => g => x => f(g(x));
  const apply = f => a => f(a);
  const flip = f => b => a => f(a) (b);
  const concat = xs => y => xs.concat(y);
  const afrom = apply(Array.from);
  const createSet = xs => new Set(xs);
  const filter = f => xs => xs.filter(apply(f));

  const dedupe = comp(afrom) (createSet);

  const union = xs => ys => {
    const zs = createSet(xs);  
    return concat(xs) (
      filter(x => zs.has(x)
       ? false
       : zs.add(x)
    ) (ys));
  }

  return union(dedupe(arr1)) (arr2)
}



// -------------
// TEST
// -------------

var array1 = ["Vijendra","Singh"];
var array2 = ["Singh", "Shakya"];

[A,B,C,D,E,G,H,J,L,M].forEach(f=> {
  console.log(`${f.name} [${f([...array1],[...array2])}]`);
})
<script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.20/lodash.min.js" integrity="sha512-90vH1Z83AJY9DmlWa8WkjkV79yfS2n2Oxhsi2dZbIv0nC4E6m5AbH8Nh156kkM7JePmqD6tcZsfad1ueoaovww==" crossorigin="anonymous"></script>
  
This snippet only presents functions used in performance tests - it not perform tests itself!

And here are example test run for chrome

enter image description here

UPDATE

I remove cases F,I,K because they modify input arrays and benchmark gives wrong results

For ES6, just one line:

a = [1, 2, 3, 4]
b = [4, 5]
[...new Set(a.concat(b))]  // [1, 2, 3, 4, 5]

There are so many solutions for merging two arrays. They can be divided into two main categories(except the use of 3rd party libraries like lodash or underscore.js).

a) combine two arrays and remove duplicated items.

b) filter out items before combining them.

Combine two arrays and remove duplicated items

Combining

// mutable operation(array1 is the combined array)
array1.push(...array2);
array1.unshift(...array2);

// immutable operation
const combined = array1.concat(array2);
const combined = [...array1, ...array2];    // ES6

Unifying

There are many ways to unifying an array, I personally suggest below two methods.

// a little bit tricky
const merged = combined.filter((item, index) => combined.indexOf(item) === index);
const merged = [...new Set(combined)];

Filter out items before combining them

There are also many ways, but I personally suggest the below code due to its simplicity.

const merged = array1.concat(array2.filter(secItem => !array1.includes(secItem)));
array1.concat(array2).filter((value, pos, arr)=>arr.indexOf(value)===pos)

The nice thing about this one is performance and that you in general, when working with arrays, are chaining methods like filter, map, etc so you can add that line and it will concat and deduplicate array2 with array1 without needing a reference to the later one (when you are chaining methods you don't have), example:

someSource()
.reduce(...)
.filter(...)
.map(...) 
// and now you want to concat array2 and deduplicate:
.concat(array2).filter((value, pos, arr)=>arr.indexOf(value)===pos)
// and keep chaining stuff
.map(...)
.find(...)
// etc

(I don't like to pollute Array.prototype and that would be the only way of respect the chain - defining a new function will break it - so I think something like this is the only way of accomplish that)

DeDuplicate single or Merge and DeDuplicate multiple array inputs. Example below.

useing ES6 - Set, for of, destructuring

I wrote this simple function which takes multiple array arguments. Does pretty much the same as the solution above it just have more practical use case. This function doesn't concatenate duplicate values in to one array only so that it can delete them at some later stage.

SHORT FUNCTION DEFINITION ( only 9 lines )

/**
* This function merging only arrays unique values. It does not merges arrays in to array with duplicate values at any stage.
*
* @params ...args Function accept multiple array input (merges them to single array with no duplicates)
* it also can be used to filter duplicates in single array
*/
function arrayDeDuplicate(...args){
   let set = new Set(); // init Set object (available as of ES6)
   for(let arr of args){ // for of loops through values
      arr.map((value) => { // map adds each value to Set object
         set.add(value); // set.add method adds only unique values
      });
   }
   return [...set]; // destructuring set object back to array object
   // alternativly we culd use:  return Array.from(set);
}

USE EXAMPLE CODEPEN:

// SCENARIO 
let a = [1,2,3,4,5,6];
let b = [4,5,6,7,8,9,10,10,10];
let c = [43,23,1,2,3];
let d = ['a','b','c','d'];
let e = ['b','c','d','e'];

// USEAGE
let uniqueArrayAll = arrayDeDuplicate(a, b, c, d, e);
let uniqueArraySingle = arrayDeDuplicate(b);

// OUTPUT
console.log(uniqueArrayAll); // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 43, 23, "a", "b", "c", "d", "e"]
console.log(uniqueArraySingle); // [4, 5, 6, 7, 8, 9, 10]

var array1 = ["one","two"];
var array2 = ["two", "three"];
var collectionOfTwoArrays = [...array1, ...array2];    
var uniqueList = array => [...new Set(array)];
console.log('Collection :');
console.log(collectionOfTwoArrays);    
console.log('Collection without duplicates :');
console.log(uniqueList(collectionOfTwoArrays));

Here an option for objects with object arrays:

const a = [{param1: "1", param2: 1},{param1: "2", param2: 2},{param1: "4", param2: 4}]
const b = [{param1: "1", param2: 1},{param1: "4", param2: 5}]


var result = a.concat(b.filter(item =>
         !JSON.stringify(a).includes(JSON.stringify(item))
    ));

console.log(result);
//Result [{param1: "1", param2: 1},{param1: "2", param2: 2},{param1: "4", param2: 4},{param1: "4", param2: 5}]
var arr1 = [1, 3, 5, 6];
var arr2 = [3, 6, 10, 11, 12];
arr1.concat(arr2.filter(ele => !arr1.includes(ele)));
console.log(arr1);

output :- [1, 3, 5, 6, 10, 11, 12]

You can try this:

const union = (a, b) => Array.from(new Set([...a, ...b]));

console.log(union(["neymar","messi"], ["ronaldo","neymar"]));

The simplest solution with filter:

var array1 = ["Vijendra","Singh"];
var array2 = ["Singh", "Shakya"];

var mergedArrayWithoutDuplicates = array1.concat(
  array2.filter(seccondArrayItem => !array1.includes(seccondArrayItem))
);

Modular, General

This could be achieved by composing two essential functions.

const getUniqueMerge = (...arrs) => getUniqueArr(mergeArrs(...arrs))
const getUniqueArr = (array) => Array.from(new Set(array))  
const mergeArrs = (...arrs) => [].concat(...arrs)

It can handle unlimited arrays, or values

console.log(getUniqueMerge(["Vijendra","Singh"],["Singh", "Shakya"])
// ["Vijendra", "Singh", "Shakya"]

console.log(getUniqueMerge(["Sheldon", "Cooper"], ["and", "Cooper", "Amy", "and"], "Farrah", "Amy", "Fowler"))
// ["Sheldon", "Cooper", "and", "Amy", "Farrah", "Fowler"]

you can use new Set to remove duplication

[...new Set([...array1 ,...array2])]

You can use loadash unionWith - _.unionWith([arrays], [comparator])

This method is like _.union except that it accepts comparator which is invoked to compare elements of arrays. Result values are chosen from the first array in which the value occurs. The comparator is invoked with two arguments: (arrVal, othVal).

var array1 = ["Vijendra","Singh"];
var array2 = ["Singh", "Shakya"];
 
var array3 = _.unionWith(array1, array2, _.isEqual);
console.log(array3);
<script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.11/lodash.min.js"></script>

If you do not want duplicates of a specific property (for example the ID)

let noDuplicate = array1.filter ( i => array2.findIndex(a => i.id==a.id)==-1 );
let result = [...noDuplicate, ...array2];

Reduce them !!!

This alternative instead of merging and deduplicating explicitly it will take one array and reduce it with another array so that each value of the first array can be iterated and destructured in an accumulative behavior, ignoring the already included values by exploiting the persistence of the array because of the recursiveness.

array2.reduce(reducer, array1.reduce(reducer, []))

Test Example:

var array1 = ["Vijendra","Singh","Singh"];
var array2 = ["Singh", "Shakya", "Shakya"];
const reducer = (accumulator, currentValue) => accumulator.includes(currentValue) ? accumulator : [...accumulator, currentValue];

console.log(
  array2.reduce(reducer, array1.reduce(reducer, []))
);

// a reduce on first array is needed to ensure a deduplicated array used as initial value on the second array being reduced

Conclusion

By far more elegant and useful when boring for-each approach wants to be avoided (not that it is not useful).

Deals with the concat() limitations on deduplication.

No need for external libraries like Underscore.js, JQuery or Lo-Dash, nor the trouble to create any built-in function to achieve the desired merged and deduplicated effect.

Oh, and HEY!, it can be done as a one-liner!!!


This answered was possible thanks to ES5 (ECMAScript 2015), beautiful include() and gorgeous reduce().

If you merging object arrays, consider use of lodash UnionBy function, it allows you to set custom predicate compare objects:

import { unionBy } from 'lodash';

const a = [{a: 1, b: 2}];
const b = [{a: 1, b: 3}];
const c = [{a: 2, b: 4}];

const result = UnionBy(a,b,c, x => x.a);

Result is: [{ a: 1; b: 2 }, { a: 2; b: 4 }]

First passed match from arrays is used in result

Here is another neat solution using Set:

const o1 = {a: 1};
const arr1 = ['!@#$%^&*()', 'gh', 123, o1, 1, true, undefined, null];
const arr2 = ['!@#$%^&*()', 123, 'abc', o1, 0x001, true, void 0, 0];

const mergeUnique = (...args) => [ ...new Set([].concat(...args)) ];

console.log(mergeUnique(arr1, arr2));

Care about efficiency, yet want to do it inline

const s = new Set(array1);
array2.forEach(a => s.add(a));
const merged_array = [...s]; // optional: convert back in array type

ES2019

You can use it like union(array1, array2, array3, ...)

/**
 * Merges two or more arrays keeping unique items. This method does
 * not change the existing arrays, but instead returns a new array.
 */
function union<T>(...arrays: T[]) {
  return [...new Set([...arrays].flat())];
}

It is ES2019 because of the flat() function, but you can use core-js to get it as a polyfill. The T here is TypeScript generic type, that you can remove if you are not using TypeScript. If you are using TypeScript, make sure to add "lib": ["es2019.array"] to compiler options in tsconfig.json.

or...

just use lodash _.union

For n arrays, you can get the union like so.

function union(arrays) {
    return new Set(arrays.flat()).keys();
};

Built a tester to check just how fast some of the performance oriented answers are. Feel free to add some more. So far, Set is both the simplest and fastest option (by bigger margins as the number of records increases), at least with simple Number types.

const records = 10000, //max records per array
  max_int = 100, //max integer value per array
  dup_rate = .5; //rate of duplication
let perf = {}, //performance logger,
  ts = 0,
  te = 0,
  array1 = [], //init arrays
  array2 = [],
  array1b = [],
  array2b = [],
  a = [];

//populate randomized arrays
for (let i = 0; i < records; i++) {
  let r = Math.random(),
    n = r * max_int;
  if (Math.random() < .5) {
    array1.push(n);
    r < dup_rate && array2.push(n);
  } else {
    array2.push(n);
    r < dup_rate && array1.push(n);
  }
}
//simple deep copies short of rfdc, in case someone wants to test with more complex data types
array1b = JSON.parse(JSON.stringify(array1));
array2b = JSON.parse(JSON.stringify(array2));
console.log('Records in Array 1:', array1.length, array1b.length);
console.log('Records in Array 2:', array2.length, array2b.length);

//test method 1 (jsperf per @Pitouli)
ts = performance.now();
for (let i = 0; i < array2.length; i++)
  if (array1.indexOf(array2[i]) === -1)
    array1.push(array2[i]); //modifies array1
te = performance.now();
perf.m1 = te - ts;
console.log('Method 1 merged', array1.length, 'records in:', perf.m1);
array1 = JSON.parse(JSON.stringify(array1b)); //reset array1

//test method 2 (classic forEach)
ts = performance.now();
array2.forEach(v => array1.includes(v) ? null : array1.push(v)); //modifies array1
te = performance.now();
perf.m2 = te - ts;
console.log('Method 2 merged', array1.length, 'records in:', perf.m2);

//test method 3 (Simplest native option)
ts = performance.now();
a = [...new Set([...array1, ...array2])]; //does not modify source arrays
te = performance.now();
perf.m3 = te - ts;
console.log('Method 3 merged', a.length, 'records in:', perf.m3);

//test method 4 (Selected Answer)
ts = performance.now();
a = array1.concat(array2); //does not modify source arrays
for (let i = 0; i < a.length; ++i) {
  for (let j = i + 1; j < a.length; ++j) {
    if (a[i] === a[j])
      a.splice(j--, 1);
  }
}
te = performance.now();
perf.m4 = te - ts;
console.log('Method 4 merged', a.length, 'records in:', perf.m4);

//test method 5 (@Kamil Kielczewski)
ts = performance.now();

function K(arr1, arr2) {
  let r = [],
    h = {};

  while (arr1.length) {
    let e = arr1.shift(); //modifies array1
    if (!h[e]) h[e] = 1 && r.push(e);
  }

  while (arr2.length) {
    let e = arr2.shift(); //modifies array2
    if (!h[e]) h[e] = 1 && r.push(e);
  }

  return r;
}
a = K(array1, array2);
te = performance.now();
perf.m5 = te - ts;
console.log('Method 5 merged', a.length, 'records in:', perf.m4);
array1 = JSON.parse(JSON.stringify(array1b)); //reset array1
array2 = JSON.parse(JSON.stringify(array2b)); //reset array2


for (let i = 1; i < 6; i++) {
  console.log('Method:', i, 'speed is', (perf['m' + i] / perf.m1 * 100).toFixed(2), '% of Method 1');
}

To offer something simpler and more elegant, in this day and age, using an existing library:

import {pipe, concat, distinct} from 'iter-ops';

// our inputs:
const array1 = ['Vijendra', 'Singh'];
const array2 = ['Singh', 'Shakya'];

const i = pipe(
    array1,
    concat(array2), // adding array
    distinct() // making it unique
);

console.log([...i]); //=> ['Vijendra', 'Singh', 'Shakya']

It is both high performance, as we are iterating only once, and the code is very easy to read.

P.S. I'm the author of iter-ops.

const array3 = array1.filter(t=> !array2.includes(t)).concat(array2)
var a = [1,2,3]
var b = [1,2,4,5]

I like one liners. This will push distinct b elements to a

b.forEach(item => a.includes(item) ? null : a.push(item));

And another version that will not modify a

var c = a.slice();
b.forEach(item => c.includes(item) ? null : c.push(item));

If you're purely using underscore.js, it doesn't have unionWith, unionBy

you can try out : _.uniq(_.union(arr1, arr2), (obj) => obj.key) ( key is the key param of each object ) this should help to get unique after union of both arrays.

I think this works faster.

removeDup = a => {

    for (let i = a.length - 1; i >= 0; i--) {
        for (let j = i-1; j >= 0; j--) {
            if (a[i] === a[j])
                a.splice(j--, 1);
        }
    }

    return a;
}
   //1.merge two array into one array

   var arr1 = [0, 1, 2, 4];
   var arr2 = [4, 5, 6];

   //for merge array we use "Array.concat"

   let combineArray = arr1.concat(arr2); //output

   alert(combineArray); //now out put is 0,1,2,4,4,5,6 but 4 reapeat

   //2.same thing with "Spread Syntex"

   let spreadArray = [...arr1, ...arr2];

   alert(spreadArray);  //now out put is 0,1,2,4,4,5,6 but 4 reapete


   /*
       if we need remove duplicate element method use are
       1.Using set
       2.using .filter
       3.using .reduce
   */

Not Performant if you have extremely large lists, and this isnt for merging since many solutions already have been documented, but i solved my problems with this solution (since most solutions of array filtering apply to simple arrays)

const uniqueVehiclesServiced = 
  invoice.services.sort().filter(function(item, pos, ary) {
    const firstIndex = invoice.services.findIndex((el, i, arr) => el.product.vin === item.product.vin)

  return !pos || firstIndex == pos;
});

I have a similar request but it is with Id of the elements in the array.

And, here is the way I do the deduplication.

It is simple, easy to maintain, and good to use.

// Vijendra's Id = Id_0
// Singh's Id = Id_1
// Shakya's Id = Id_2

let item0 = { 'Id': 'Id_0', 'value': 'Vijendra' };
let item1 = { 'Id': 'Id_1', 'value': 'Singh' };
let item2 = { 'Id': 'Id_2', 'value': 'Shakya' };

let array = [];

array = [ item0, item1, item1, item2 ];

let obj = {};
array.forEach(item => {
    obj[item.Id] = item;
});

let deduplicatedArray = [];
let deduplicatedArrayOnlyValues = [];
for(let [index, item] of Object.values(obj).entries()){
    deduplicatedArray = [ ...deduplicatedArray, item ];
    deduplicatedArrayOnlyValues = [ ...deduplicatedArrayOnlyValues , item.value ];
};
    
console.log( JSON.stringify(array) );
console.log( JSON.stringify(deduplicatedArray) );
console.log( JSON.stringify(deduplicatedArrayOnlyValues ) );

The console log

[{"recordId":"Id_0","value":"Vijendra"},{"recordId":"Id_1","value":"Singh"},{"recordId":"Id_1","value":"Singh"},{"recordId":"Id_2","value":"Shakya"}]

[{"recordId":"Id_0","value":"Vijendra"},{"recordId":"Id_1","value":"Singh"},{"recordId":"Id_2","value":"Shakya"}]

["Vijendra","Singh","Shakya"]
  1. Using array.concat() and array.filter()
  2. Using new Set object and Spread Operator
  3. Using array.concat and new Set object

let array1 = [1, 2, 3, 4, 5]
let array2 = [1, 4, 6, 9]

// Using array.concat and array.filter
const array3 = array1.concat(array2.filter((item)=> array1.indexOf(item) == -1 ))
console.log('array3 : ', array3);

// Using new Set and Spread Operator
const array4 = [...new Set([...array1 ,...array2])];
console.log('array4 : ', array4);

// Using array.concat and new Set
const array5 = [...new Set(array1.concat(array2))];
console.log('array5 : ', array5);

Given two sorted arrays of simple types without duplicates, this will merge them in O(n) time, and the output will be also sorted.

function merge(a, b) {
    let i=0;
    let j=0;
    let c = [];
    for (;;) {
        if (i == a.length) {
            if (j == b.length) return c;
            c.push(b[j++]);
        } else if (j == b.length || a[i] < b[j]) {
            c.push(a[i++]);
        } else {
            if (a[i] == b[j]) ++i;   // skip duplicates
            c.push(b[j++]);
        }
    }
}

ES 6 Version

Try this... this should resolve your problem

var array1 = ["Vijendra","Singh"]; var array2 = ["Singh", "Shakya"];

var output = [...new Set([...array1,...array2])]

console.log('Merged Array', output)

Related