Remove duplicates from an array of objects in JavaScript


I have an object that contains an array of objects.

things = new Object();

things.thing = new Array();

things.thing.push({place:"here",name:"stuff"});
things.thing.push({place:"there",name:"morestuff"});
things.thing.push({place:"there",name:"morestuff"});

I'm wondering what is the best method to remove duplicate objects from an array. So for example, things.thing would become...

{place:"here",name:"stuff"},
{place:"there",name:"morestuff"}

A primitive method would be:

var obj = {};

for ( var i=0, len=things.thing.length; i < len; i++ )
    obj[things.thing[i]['place']] = things.thing[i];

things.thing = new Array();
for ( var key in obj )
    things.thing.push(obj[key]);

How about with some es6 magic?

things.thing = things.thing.filter((thing, index, self) =>
  index === self.findIndex((t) => (
    t.place === thing.place && t.name === thing.name
  ))
)

Reference URL

A more generic solution would be:

const uniqueArray = things.thing.filter((thing, index) => {
  const _thing = JSON.stringify(thing);
  return index === things.thing.findIndex(obj => {
    return JSON.stringify(obj) === _thing;
  });
});

Stackblitz Example


If you can use Javascript libraries such as underscore or lodash, I recommend having a look at _.uniq function in their libraries. From lodash:

_.uniq(array, [isSorted=false], [callback=_.identity], [thisArg])

Basically, you pass in the array that in here is an object literal and you pass in the attribute that you want to remove duplicates with in the original data array, like this:

var data = [{'name': 'Amir', 'surname': 'Rahnama'}, {'name': 'Amir', 'surname': 'Stevens'}];
var non_duplidated_data = _.uniq(data, 'name'); 

UPDATE: Lodash now has introduced a .uniqBy as well.


I had this exact same requirement, to remove duplicate objects in a array, based on duplicates on a single field. I found the code here: Javascript: Remove Duplicates from Array of Objects

So in my example, I'm removing any object from the array that has a duplicate licenseNum string value.

var arrayWithDuplicates = [
    {"type":"LICENSE", "licenseNum": "12345", state:"NV"},
    {"type":"LICENSE", "licenseNum": "A7846", state:"CA"},
    {"type":"LICENSE", "licenseNum": "12345", state:"OR"},
    {"type":"LICENSE", "licenseNum": "10849", state:"CA"},
    {"type":"LICENSE", "licenseNum": "B7037", state:"WA"},
    {"type":"LICENSE", "licenseNum": "12345", state:"NM"}
];

function removeDuplicates(originalArray, prop) {
     var newArray = [];
     var lookupObject  = {};

     for(var i in originalArray) {
        lookupObject[originalArray[i][prop]] = originalArray[i];
     }

     for(i in lookupObject) {
         newArray.push(lookupObject[i]);
     }
      return newArray;
 }

var uniqueArray = removeDuplicates(arrayWithDuplicates, "licenseNum");
console.log("uniqueArray is: " + JSON.stringify(uniqueArray));

The results:

uniqueArray is:

[{"type":"LICENSE","licenseNum":"10849","state":"CA"},
{"type":"LICENSE","licenseNum":"12345","state":"NM"},
{"type":"LICENSE","licenseNum":"A7846","state":"CA"},
{"type":"LICENSE","licenseNum":"B7037","state":"WA"}]

One liner using Set

var things = new Object();

things.thing = new Array();

things.thing.push({place:"here",name:"stuff"});
things.thing.push({place:"there",name:"morestuff"});
things.thing.push({place:"there",name:"morestuff"});

// assign things.thing to myData for brevity
var myData = things.thing;

things.thing = Array.from(new Set(myData.map(JSON.stringify))).map(JSON.parse);

console.log(things.thing)

Explanation:

  1. new Set(myData.map(JSON.stringify)) creates a Set object using the stringified myData elements.
  2. Set object will ensure that every element is unique.
  3. Then I create an array based on the elements of the created set using Array.from.
  4. Finally, I use JSON.parse to convert stringified element back to an object.

Here's another option to do it using Array iterating methods if you need comparison only by one field of an object:

    function uniq(a, param){
        return a.filter(function(item, pos, array){
            return array.map(function(mapItem){ return mapItem[param]; }).indexOf(item[param]) === pos;
        })
    }

    uniq(things.thing, 'place');

Shortest one liners for ES6+

Find unique id's in an array.

arr.filter((v,i,a)=>a.findIndex(t=>(t.id === v.id))===i)

Unique by multiple properties ( place and name )

arr.filter((v,i,a)=>a.findIndex(t=>(t.place === v.place && t.name===v.name))===i)

Unique by all properties (This will be slow for large arrays)

arr.filter((v,i,a)=>a.findIndex(t=>(JSON.stringify(t) === JSON.stringify(v)))===i)

Keep the last occurrence. Add slice and reverse to beginning and reverse again in the end.

arr.slice().reverse().filter((v,i,a)=>a.findIndex(t=>(t.id === v.id))===i).reverse()

If you can wait to eliminate the duplicates until after all the additions, the typical approach is to first sort the array and then eliminate duplicates. The sorting avoids the N * N approach of scanning the array for each element as you walk through them.

The "eliminate duplicates" function is usually called unique or uniq. Some existing implementations may combine the two steps, e.g., prototype's uniq

This post has few ideas to try (and some to avoid :-) ) if your library doesn't already have one! Personally I find this one the most straight forward:

    function unique(a){
        a.sort();
        for(var i = 1; i < a.length; ){
            if(a[i-1] == a[i]){
                a.splice(i, 1);
            } else {
                i++;
            }
        }
        return a;
    }  

    // Provide your own comparison
    function unique(a, compareFunc){
        a.sort( compareFunc );
        for(var i = 1; i < a.length; ){
            if( compareFunc(a[i-1], a[i]) === 0){
                a.splice(i, 1);
            } else {
                i++;
            }
        }
        return a;
    }

one liner is here

let arr = [
  {id:1,name:"sravan ganji"},
  {id:2,name:"anu"},
  {id:4,name:"mammu"},
  {id:3,name:"sanju"},
  {id:3,name:"ram"},
];

console.log(Object.values(arr.reduce((acc,cur)=>Object.assign(acc,{[cur.id]:cur}),{})))


The simplest way is use filter:

var uniq = {}
var arr  = [{"id":"1"},{"id":"1"},{"id":"2"}]
var arrFiltered = arr.filter(obj => !uniq[obj.id] && (uniq[obj.id] = true));
console.log('arrFiltered', arrFiltered)


Using ES6 in a single line you can get a unique list of objects by key:

const unique = [...new Map(arr.map(item => [item[key], item])).values()]

It can be put into a function and used like in the below example:

const arr = [
  {place: "here", name: "stuff"},
  {place: "there", name: "morestuff"},
  {place: "a", name: "morestuff"},
  {place: "b", name: "morestuff"},
  {place: "c", name: "morestuff"},
  {place: "here", name: "lol"},
  {place: "there", name: "test"}
]

function getUniqueListBy(arr, key) {
    return [...new Map(arr.map(item => [item[key], item])).values()]
}

const arr1 = getUniqueListBy(arr, 'place')

console.log(arr1)


const arr2 = getUniqueListBy(arr, 'name')

console.log(arr2)


This is a generic way of doing this: you pass in a function that tests whether two elements of an array are considered equal. In this case, it compares the values of the name and place properties of the two objects being compared.

ES5 answer

function removeDuplicates(arr, equals) {
    var originalArr = arr.slice(0);
    var i, len, j, val;
    arr.length = 0;

    for (i = 0, len = originalArr.length; i < len; ++i) {
        val = originalArr[i];
        if (!arr.some(function(item) { return thingsEqual(item, val); })) {
            arr.push(val);
        }
    }
}

function thingsEqual(thing1, thing2) {
    return thing1.place === thing2.place
        && thing1.name === thing2.name;
}

var things = [
  {place:"here",name:"stuff"},
  {place:"there",name:"morestuff"},
  {place:"there",name:"morestuff"}
];

removeDuplicates(things, thingsEqual);
console.log(things);

Original ES3 answer

function arrayContains(arr, val, equals) {
    var i = arr.length;
    while (i--) {
        if ( equals(arr[i], val) ) {
            return true;
        }
    }
    return false;
}

function removeDuplicates(arr, equals) {
    var originalArr = arr.slice(0);
    var i, len, j, val;
    arr.length = 0;

    for (i = 0, len = originalArr.length; i < len; ++i) {
        val = originalArr[i];
        if (!arrayContains(arr, val, equals)) {
            arr.push(val);
        }
    }
}

function thingsEqual(thing1, thing2) {
    return thing1.place === thing2.place
        && thing1.name === thing2.name;
}

removeDuplicates(things.thing, thingsEqual);

To add one more to the list. Using ES6 and Array.reduce with Array.find.
In this example filtering objects based on a guid property.

let filtered = array.reduce((accumulator, current) => {
  if (! accumulator.find(({guid}) => guid === current.guid)) {
    accumulator.push(current);
  }
  return accumulator;
}, []);

Extending this one to allow selection of a property and compress it into a one liner:

const uniqify = (array, key) => array.reduce((prev, curr) => prev.find(a => a[key] === curr[key]) ? prev : prev.push(curr) && prev, []);

To use it pass an array of objects and the name of the key you wish to de-dupe on as a string value:

const result = uniqify(myArrayOfObjects, 'guid')

You could also use a Map:

const dedupThings = Array.from(things.thing.reduce((m, t) => m.set(t.place, t), new Map()).values());

Full sample:

const things = new Object();

things.thing = new Array();

things.thing.push({place:"here",name:"stuff"});
things.thing.push({place:"there",name:"morestuff"});
things.thing.push({place:"there",name:"morestuff"});

const dedupThings = Array.from(things.thing.reduce((m, t) => m.set(t.place, t), new Map()).values());

console.log(JSON.stringify(dedupThings, null, 4));

Result:

[
    {
        "place": "here",
        "name": "stuff"
    },
    {
        "place": "there",
        "name": "morestuff"
    }
]

Dang, kids, let's crush this thing down, why don't we?

let uniqIds = {}, source = [{id:'a'},{id:'b'},{id:'c'},{id:'b'},{id:'a'},{id:'d'}];
let filtered = source.filter(obj => !uniqIds[obj.id] && (uniqIds[obj.id] = true));
console.log(filtered);
// EXPECTED: [{id:'a'},{id:'b'},{id:'c'},{id:'d'}];


A TypeScript solution

This will remove duplicate objects and also preserve the types of the objects.

function removeDuplicateObjects(array: any[]) {
  return [...new Set(array.map(s => JSON.stringify(s)))]
    .map(s => JSON.parse(s));
}

Considering lodash.uniqWith

var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }, { 'x': 1, 'y': 2 }];

_.uniqWith(objects, _.isEqual);
// => [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]

Another option would be to create a custom indexOf function, which compares the values of your chosen property for each object and wrap this in a reduce function.

var uniq = redundant_array.reduce(function(a,b){
      function indexOfProperty (a, b){
          for (var i=0;i<a.length;i++){
              if(a[i].property == b.property){
                   return i;
               }
          }
         return -1;
      }

      if (indexOfProperty(a,b) < 0 ) a.push(b);
        return a;
    },[]);

let myData = [{place:"here",name:"stuff"}, 
 {place:"there",name:"morestuff"},
 {place:"there",name:"morestuff"}];


let q = [...new Map(myData.map(obj => [JSON.stringify(obj), obj])).values()];

console.log(q)

One-liner using ES6 and new Map().

// assign things.thing to myData
let myData = things.thing;

[...new Map(myData.map(obj => [JSON.stringify(obj), obj])).values()];

Details:-

  1. Doing .map() on the data list and converting each individual object into a [key, value] pair array(length =2), the first element(key) would be the stringified version of the object and second(value) would be an object itself.
  2. Adding above created array list to new Map() would have the key as stringified object and any same key addition would result in overriding the already existing key.
  3. Using .values() would give MapIterator with all values in a Map (obj in our case)
  4. Finally, spread ... operator to give new Array with values from the above step.

Here is a solution for es6 where you only want to keep the last item. This solution is functional and Airbnb style compliant.

const things = {
  thing: [
    { place: 'here', name: 'stuff' },
    { place: 'there', name: 'morestuff1' },
    { place: 'there', name: 'morestuff2' }, 
  ],
};

const removeDuplicates = (array, key) => {
  return array.reduce((arr, item) => {
    const removed = arr.filter(i => i[key] !== item[key]);
    return [...removed, item];
  }, []);
};

console.log(removeDuplicates(things.thing, 'place'));
// > [{ place: 'here', name: 'stuff' }, { place: 'there', name: 'morestuff2' }]

removeDuplicates() takes in an array of objects and returns a new array without any duplicate objects (based on the id property).

const allTests = [
  {name: 'Test1', id: '1'}, 
  {name: 'Test3', id: '3'},
  {name: 'Test2', id: '2'},
  {name: 'Test2', id: '2'},
  {name: 'Test3', id: '3'}
];

function removeDuplicates(array) {
  let uniq = {};
  return array.filter(obj => !uniq[obj.id] && (uniq[obj.id] = true))
}

removeDuplicates(allTests);

Expected outcome:

[
  {name: 'Test1', id: '1'}, 
  {name: 'Test3', id: '3'},
  {name: 'Test2', id: '2'}
];

First, we set value of variable uniq to an empty object.

Next, we filter through the array of objects. Filter creates a new array with all elements that pass the test implemented by the provided function.

return array.filter(obj => !uniq[obj.id] && (uniq[obj.id] = true));

Above, we use the short-circuiting functionality of &&. If the left side of the && evaluates to true, then it returns the value on the right of the &&. If the left side is false, it returns what is on the left side of the &&.

For each object(obj) we check uniq for a property named the value of obj.id (In this case, on the first iteration it would check for the property '1'.) We want the opposite of what it returns (either true or false) which is why we use the ! in !uniq[obj.id]. If uniq has the id property already, it returns true which evaluates to false (!) telling the filter function NOT to add that obj. However, if it does not find the obj.id property, it returns false which then evaluates to true (!) and returns everything to the right of the &&, or (uniq[obj.id] = true). This is a truthy value, telling the filter method to add that obj to the returned array, and it also adds the property {1: true} to uniq. This ensure that any other obj instance with that same id will not be added again.


let data = [
  {
    'name': 'Amir',
    'surname': 'Rahnama'
  }, 
  {
    'name': 'Amir',
    'surname': 'Stevens'
  }
];
let non_duplicated_data = _.uniqBy(data, 'name');

Multiple Ways to remove duplicate objects from array

let nameList = [{
  "id" : 5,
  "name" : "Johnson"
},{
  "id" : 4,
  "name" : "John"
},{
  "id" : 3,
  "name" : "Juliet"
},{
  "id" : 2,
  "name" : "Nancy"
},{
  "id" : 5,
  "name" : "Johnson"
},{
  "id" : 5,
  "name" : "Johnson"
},{
  "id": 2,
  "name": 'Nancy'
}];


let output1 = Array.from(new Set(nameList.map(list=>list.id))).map(id=>{
  return {
   id: id,
   name: nameList.find(a=>a.id===id).name
  }
})

console.log('----------------------------1st way')
console.log(output1)

let output2 = nameList.reduce((accumulator, element) => {
     if (!accumulator.find(el => el['id'] === element['id'])) {
          accumulator.push(element);
      }
     return accumulator;
   },[]);
console.log('----------------------------2nd way')
console.log(output2)


const output3 = [];
const map = new Map();
for (const object of nameList) {
    if(!map.has(object.id)){
        map.set(object.id, true);
        output3.push({
            id: object.id,
            name: object.name
        });
    }
}
console.log('----------------------------3rd way')
console.log(output3)

let output4 = [...new Set(nameList.map(s => JSON.stringify(s)))]
    .map(s => JSON.parse(s));
    
console.log('----------------------------4th way')
console.log(output4);


es6 magic in one line... readable at that!

// returns the union of two arrays where duplicate objects with the same 'prop' are removed
const removeDuplicatesWith = (a, b, prop) => a.filter(x => !b.find(y => x[prop] === y[prop]);

Simple solution with ES6 'reduce' and 'find' array helper methods

Works efficiently and perfectly fine!

"use strict";

var things = new Object();
things.thing = new Array();
things.thing.push({
    place: "here",
    name: "stuff"
});
things.thing.push({
    place: "there",
    name: "morestuff"
});
things.thing.push({
    place: "there",
    name: "morestuff"
});

// the logic is here

function removeDup(something) {
    return something.thing.reduce(function (prev, ele) {
        var found = prev.find(function (fele) {
            return ele.place === fele.place && ele.name === fele.name;
        });
        if (!found) {
            prev.push(ele);
        }
        return prev;
    }, []);
}
console.log(removeDup(things));

I believe a combination of reduce with JSON.stringify to perfectly compare Objects and selectively adding those who are not already in the accumulator is an elegant way.

Keep in mind that JSON.stringify might become a performance issue in extreme cases where the array has many Objects and they are complex, BUT for majority of the time, this is the shortest way to go IMHO.

var collection= [{a:1},{a:2},{a:1},{a:3}]

var filtered = collection.reduce((filtered, item) => {
  if( !filtered.some(filteredItem => JSON.stringify(filteredItem) == JSON.stringify(item)) )
    filtered.push(item)
  return filtered
}, [])

console.log(filtered)

Another way of writing the same (but less efficient):

collection.reduce((filtered, item) => 
  filtered.some(filteredItem => 
    JSON.stringify(filteredItem ) == JSON.stringify(item)) 
      ? filtered
      : [...filtered, item]
, [])

If you don't mind your unique array being sorted afterwards, this would be an efficient solution:

things.thing
  .sort(((a, b) => a.place < b.place)
  .filter((current, index, array) =>
    index === 0 || current.place !== array[index - 1].place)

This way, you only have to compare the current element with the previous element in the array. Sorting once before filtering (O(n*log(n))) is cheaper than searching for a duplicate in the entire array for every array element (O(n²)).


This is simple way how to remove duplicity from array of objects.

I work with data a lot and this is useful for me.

const data = [{name: 'AAA'}, {name: 'AAA'}, {name: 'BBB'}, {name: 'AAA'}];
function removeDuplicity(datas){
    return datas.filter((item, index,arr)=>{
    const c = arr.map(item=> item.name);
    return  index === c.indexOf(item.name)
  })
}

console.log(removeDuplicity(data))

will print into console :

[[object Object] {
name: "AAA"
}, [object Object] {
name: "BBB"
}]

str =[
{"item_id":1},
{"item_id":2},
{"item_id":2}
]

obj =[]
for (x in str){
    if(check(str[x].item_id)){
        obj.push(str[x])
    }   
}
function check(id){
    flag=0
    for (y in obj){
        if(obj[y].item_id === id){
            flag =1
        }
    }
    if(flag ==0) return true
    else return false

}
console.log(obj)

str is an array of objects. There exists objects having same value (here a small example, there are two objects having same item_id as 2). check(id) is a function that checks if any object having same item_id exists or not. if it exists return false otherwise return true. According to that result, push the object into a new array obj The output of the above code is [{"item_id":1},{"item_id":2}]


Have you heard of Lodash library? I recommend you this utility, when you don't really want to apply your logic to the code, and use already present code which is optimised and reliable.

Consider making an array like this

things.thing.push({place:"utopia",name:"unicorn"});
things.thing.push({place:"jade_palace",name:"po"});
things.thing.push({place:"jade_palace",name:"tigress"});
things.thing.push({place:"utopia",name:"flying_reindeer"});
things.thing.push({place:"panda_village",name:"po"});

Note that if you want to keep one attribute unique, you may very well do that by using lodash library. Here, you may use _.uniqBy

.uniqBy(array, [iteratee=.identity])

This method is like _.uniq (which returns a duplicate-free version of an array, in which only the first occurrence of each element is kept) except that it accepts iteratee which is invoked for each element in array to generate the criterion by which uniqueness is computed.

So, for example, if you want to return an array having unique attribute of 'place'

_.uniqBy(things.thing, 'place')

Similarly, if you want unique attribute as 'name'

_.uniqBy(things.thing, 'name')

Hope this helps.

Cheers!


If you don't want to specify a list of properties:

function removeDuplicates(myArr) {
  var props = Object.keys(myArr[0])
  return myArr.filter((item, index, self) =>
    index === self.findIndex((t) => (
      props.every(prop => {
        return t[prop] === item[prop]
      })
    ))
  )
}

OBS! Not compatible with IE11.


Continuing exploring ES6 ways of removing duplicates from array of objects: setting thisArg argument of Array.prototype.filter to new Set provides a decent alternative:

const things = [
  {place:"here",name:"stuff"},
  {place:"there",name:"morestuff"},
  {place:"there",name:"morestuff"}
];

const filtered = things.filter(function({place, name}) {

  const key =`${place}${name}`;

  return !this.has(key) && this.add(key);

}, new Set);

console.log(filtered);

However, it will not work with arrow functions () =>, as this is bound to their lexical scope.


This way work well for me .

function arrayUnique(arr, uniqueKey) {
  const flagList = new Set()
  return arr.filter(function(item) {
    if (!flagList.has(item[uniqueKey])) {
      flagList.add(item[uniqueKey])
      return true
    }
  })
}
const data = [
  {
    name: 'Kyle',
    occupation: 'Fashion Designer'
  },
  {
    name: 'Kyle',
    occupation: 'Fashion Designer'
  },
  {
    name: 'Emily',
    occupation: 'Web Designer'
  },
  {
    name: 'Melissa',
    occupation: 'Fashion Designer'
  },
  {
    name: 'Tom',
    occupation: 'Web Developer'
  },
  {
    name: 'Tom',
    occupation: 'Web Developer'
  }
]
console.table(arrayUnique(data, 'name'))// work well

printout

????????????????????????????????????????????
? (index) ?   name    ?     occupation     ?
????????????????????????????????????????????
?    0    ?  'Kyle'   ? 'Fashion Designer' ?
?    1    ?  'Emily'  ?   'Web Designer'   ?
?    2    ? 'Melissa' ? 'Fashion Designer' ?
?    3    ?   'Tom'   ?  'Web Developer'   ?
????????????????????????????????????????????

ES5:

function arrayUnique(arr, uniqueKey) {
  const flagList = []
  return arr.filter(function(item) {
    if (flagList.indexOf(item[uniqueKey]) === -1) {
      flagList.push(item[uniqueKey])
      return true
    }
  })
}

These two ways are simpler and more understandable.