remove redundant key value from json array python

user_14407604 0
Viewed 112

I have a json file contains an array of objects, the data inside the file is something like this:

[
 {‘name’: ‘A’,
 ‘address’: ‘some address related to A’,
 ‘details’: ‘some details related to A’},
 {‘name’: ‘B’,
 ‘address’: ‘some address related to A’,
 ‘details’: ‘some details related to B’},
 {‘name’: ‘C’,
 ‘address’: ‘some address related to A’,
 ‘details’: ‘some details related to C’}
]

and I want to remove redundant key value, so the output should be something like this:

  [
   {‘name’: ‘A’,
   ‘address’: ‘some address related to A’,
   ‘details’: ‘some details related to A’},
   {‘name’: ‘B’,
   ‘details’: ‘some details related to B’},
   {‘name’: ‘C’,
   ‘details’: ‘some details related to C’}
  ]

so, I've tried this code found it in this link:

import json

with open(‘./myfile.json’) as fp:
    data= fp.read()
  
unique = []
for n in data:
    if all(unique_data["address"] != data for unique_data["address"] in unique):
        unique.append(n)

#print(unique)   
with open(“./cleanedRedundancy.json”, ‘w’) as f:
     f.write(unique)

but it gives me this error:

TypeError: string indices must be integers
arrays python json
1 Answers

I did solution with/without files support, without by default, for your case to support files change use_files = False to use_files = True inside my script.

I expected that you want to remove duplicates having same (key, value) pair.

Try it online!

import json

use_files = False
# Only duplicates with next keys will be deleted
only_keys = {'address', 'complex'}

if not use_files:
    fdata = """
    [
     {
       "name": "A",
       "address": "some address related to A",
       "details": "some details related to A"
     },
     {
       "name": "B",
       "address": "some address related to A",
       "details": "some details related to B",
       "complex": ["x", {"y": "z", "p": "q"}],
       "dont_remove": "test"
     },
     {
       "name": "C",
       "address": "some address related to A",
       "details": "some details related to C",
       "complex": ["x", {"p": "q", "y": "z"}],
       "dont_remove": "test"
     }
    ]
    """

if use_files:
    with open("./myfile.json", 'r', encoding = 'utf-8') as fp:
        data = fp.read()
else:
    data = fdata

entries = json.loads(data)

unique = set()
for e in entries:
    for k, v in list(e.items()):
        if k not in only_keys:
            continue
        v = json.dumps(v, sort_keys = True)
        if (k, v) in unique:
            del e[k]
        else:
            unique.add((k, v))

if use_files:
    with open("./cleanedRedundancy.json", "w", encoding = 'utf-8') as f:
        f.write(json.dumps(entries, indent = 4, ensure_ascii = False))
else:
    print(json.dumps(entries, indent = 4, ensure_ascii = False))

Output:

[
    {
        "name": "A",
        "address": "some address related to A",
        "details": "some details related to A"
    },
    {
        "name": "B",
        "details": "some details related to B",
        "complex": [
            "x",
            {
                "y": "z",
                "p": "q"
            }
        ],
        "dont_remove": "test"
    },
    {
        "name": "C",
        "details": "some details related to C",
        "dont_remove": "test"
    }
]
user_9415310
Related