Python averaging list of lists of nested dicts

def merge_items(items):
    result = {}
    if len(items):
        result['id'] = items[0]['id']
        result['prediction'] = round(sum([item['prediction'] for item in items]) / len(items), 2)
        result['confidence'] = []
        result['confidence'].append({
            'percentile': items[0]['confidence'][0]['percentile'],
            'lower': round(sum(item['confidence'][0]['lower'] for item in items) / len(items), 2),
            'upper': round(sum(item['confidence'][0]['upper'] for item in items) / len(items), 2),
        })

    return result


result = []
ids = list(set([el['id'] for item in data for el in item]))
for id in ids:
    to_merge = [sub_item for item in data for sub_item in item if sub_item['id'] == id]
    result.append(merge_items(to_merge))

print(result)

dicc = {}

for e in l:
    for d in e:
        if d["id"] not in dicc:
            dicc[d["id"]] = {"prediction": [], "lower": [], "upper": []}

        dicc[d["id"]]["prediction"].append(d["prediction"])
        dicc[d["id"]]["lower"].append(d["confidence"][0]["lower"])
        dicc[d["id"]]["upper"].append(d["confidence"][0]["upper"])
        
        
for k in dicc:
    dicc[k]["average_prediction"] = sum(dicc[k]["prediction"])/len(dicc[k]["prediction"])
    dicc[k]["average_lower"] = sum(dicc[k]["lower"])/len(dicc[k]["lower"])
    dicc[k]["average_upper"] = sum(dicc[k]["upper"])/len(dicc[k]["upper"])

print(dicc)

{713: {'prediction': [4.8, 5.8], 'lower': [4.8, 4.2], 'upper': [5.7, 6.7], 'average_prediction': 5.3, 'average_lower': 4.5, 'average_upper': 6.2}, 714: {'prediction': [4.936893921359024, 2.936893921359024], 'lower': [4.9, 1.9], 'upper': [5.7, 3.7], 'average_prediction': 3.936893921359024, 'average_lower': 3.4000000000000004, 'average_upper': 4.7}}

You really have three parts to this question.

1. How do you unpack the lists and group by the ids in preparation for some kind of aggregation? You have lots of options, but a pretty classic one is to make a lookup table and append any new values:

groups = {}

# `data` is the outer list in your nested structure
for d in (d for L in data for d in L):
    L = groups.get(d['id'], [])
    L.append(d)
    groups[d['id']] = L

2. How do you aggregate those dictionaries so that you have an average of all the numeric values? There are lots of approaches with varying numeric stability. I'll start with an easy one that recursively walks a partial result set and a new entry.

Note that this assumes an incredibly consistent object structure (like you have shown). If you sometimes have missing keys, mismatched lengths, or other discrepancies you'll have to think long and hard about the exact details of what you want to happen when those structures are merged -- there isn't a one-size fits all solution.

def walk(avgs, new, n):
    """
    Most of this algorithm is just walking the object structure.
    We keep any keys, lists, etc the same and only average the
    numeric elements.
    """
    if isinstance(avgs, dict):
        return {k:walk(avgs[k], new[k], n) for k in avgs}
    if isinstance(avgs, list):
        return [walk(x, y, n) for x,y in zip(avgs, new)]
    if isinstance(avgs, float):  # integers and whatnot also satisfy this
        """
        This is the only place that averaging actually happens.
        At the risk of some accumulated errors, this directly
        computes the total of the last n+1 items and divides
        by n+1.
        """
        return (avgs*n+new)/(n+1.)
    return avgs

def merge(L):
    if not L:
        # never happens using the above grouping code
        return None
    d = L[0]
    for n, new in enumerate(L[1:], 1):
        d = walks(d, new, n)
    return d

averaged = {k:merge(v) for k,v in groups.items()}

You probably only want certain keys like the prediction to be averaged. You can do the filtering beforehand on the grouped objects or afterward (it's probably more efficient to do it beforehand):

# before
groups = {
    # any transformation you'd like to apply to the dictionaries
    k:[{s:d[s] for s in ('prediction', 'confidence')} for d in L] for k,L in groups.items()
}

# after
averaged = {
    # basically the same code, except there's only one object per key
    k:{s:d[s] for s in ('prediction', 'confidence')} for k,d in averaged.items()
}

For a note on efficiency, I created a bunch of intermediate lists, but those aren't really necessary. Instead of grouping then aggregating you can absolutely apply a rolling update algorithm and save some memory.

averaged = {}

# `data` is the outer list in your nested structure
for d in (d for L in data for d in L):
    key = d['id']
    d = {s:d[s] for s in ('prediction', 'confidence')}  # any desired transforms

    if key not in averaged:
        averaged[key] = (d, 1)
    else:
        agg, n = groups[key]
        averaged[key] = (walk(agg, d, n), n+1)

averaged = {k:v[0] for k,v in averaged.items()}

3. We still don't have the output formatted quite like you want (we have a dictionary, and you want a list where the keys are included in the objects). That's a pretty easy problem to solve though:

def inline_key(d, key):
    # not a pure function, but we're lazy, and the original
    # values are never used
    d['id'] = key
    return d

final_result = [inline_key(d, k) for k,d in averaged.items()]

Try this :

from copy import deepcopy

input = [[
    {
        "id": 713,
        "prediction": 4.8,
        "confidence": [
            {"percentile": "75", "lower": 4.8, "upper": 5.7}
        ],
    },
    {
        "id": 714,
        "prediction": 4.936893921359024,
        "confidence": [
            {"percentile": "75", "lower": 4.9, "upper": 5.7}
        ],
    },
],
[
    {
        "id": 713,
        "prediction": 5.8,
        "confidence": [
            {"percentile": "75", "lower": 4.2, "upper": 6.7}
        ],
    },
    {
        "id": 714,
        "prediction": 2.936893921359024,
        "confidence": [
            {"percentile": "75", "lower": 1.9, "upper": 3.7}
        ],
    },

]]

final_dict_list = []

processed_id = []

for item in input:
    for dict_ele in item:
        if dict_ele["id"] in processed_id:
            for final_item in final_dict_list:
                if final_item['id'] == dict_ele["id"]:
                    final_item["prediction"] += dict_ele["prediction"]
                    final_item["confidence"][0]["lower"] += dict_ele["confidence"][0]["lower"]
                    final_item["confidence"][0]["upper"] += dict_ele["confidence"][0]["upper"]
        else:
            final_dict = deepcopy(dict_ele)
            final_dict_list.append(final_dict)
            processed_id.append(dict_ele["id"])


numer_of_items = len(input)
for item in final_dict_list:
    item["prediction"] /= numer_of_items
    item["confidence"][0]["lower"] /= numer_of_items
    item["confidence"][0]["upper"] /= numer_of_items

print(final_dict_list)

OUTPUT :

[
{'confidence': [{'upper': 6.2, 'lower': 4.5, 'percentile': '75'}], 'id': 713, 'prediction': 5.3},
{'confidence': [{'upper': 4.7, 'lower': 3.4000000000000004, 'percentile': '75'}], 'id': 714, 'prediction': 3.936893921359024}]

Just to point, it could have been much easier if the structure of data would have been a little differently created.