How to append list of bounding box co-ordinates and text confidence score to a pandas dataframe

The desired output can be achieved by pre-processing the data to extract relevant information and then combine the datasets using .merge(). As these tasks needs to be performed for both code and qty/quantity columns, some steps are repetitive and should be refactored to enhance readability. Nevertheless, a basic approach might be:

import pandas as pd


data = {
    'type': 'table',
    'bbox': [0, 0, 349, 71],
    'res': {
        'cell_bbox': [
            [
                2.1059176921844482,
                0.7967990636825562,
                54.594913482666016,
                0.9006235599517822,
                53.41596984863281,
                16.244869232177734,
                1.93153977394104,
                15.716910362243652,
                ],
            [
                62.804359436035156,
                0.7160009145736694,
                138.24855041503906,
                0.786230206489563,
                135.5400390625,
                13.393481254577637,
                60.11547088623047,
                13.050923347473145,
                ],
            [
                98.69075775146484,
                0.7997479438781738,
                329.5711669921875,
                0.8572822213172913,
                328.5466613769531,
                17.500146865844727,
                94.86428833007812,
                17.191831588745117,
                ],
            [
                0.9269500970840454,
                17.537534713745117,
                49.044742584228516,
                17.845354080200195,
                48.904449462890625,
                41.31473159790039,
                0.9105174541473389,
                41.0366325378418,
                ],
            [
                43.86700439453125,
                17.503067016601562,
                94.20509338378906,
                17.736175537109375,
                93.4029541015625,
                40.03254318237305,
                43.05289077758789,
                39.972354888916016,
                ],
            [
                94.61170959472656,
                17.709407806396484,
                345.9893798828125,
                18.13838768005371,
                345.9151306152344,
                43.062191009521484,
                93.54864501953125,
                42.82071304321289,
                ],
            [
                0.9099741578102112,
                41.6998405456543,
                45.47168731689453,
                41.82848358154297,
                45.39813995361328,
                67.85848999023438,
                0.9060577154159546,
                67.83340454101562,
                ],
            [
                44.603145599365234,
                41.61592483520508,
                92.34540557861328,
                41.64745330810547,
                91.45874786376953,
                66.62190246582031,
                43.83229064941406,
                66.60901641845703,
                ],
            [
                92.18399810791016,
                41.781700134277344,
                344.60528564453125,
                41.8816032409668,
                344.576171875,
                66.88459777832031,
                92.13319396972656,
                66.8738784790039,
                ],
            ],
        'boxes': [
            [0., 2., 48., 25.],
            [91., 2., 276., 24.],
            [0., 24., 49., 50.],
            [53., 24., 72., 46.],
            [91., 28., 347., 49.],
            [1., 50., 49., 69.],
            [92., 51., 340., 69.],
            ],
        'rec_res': [
            ('E3509', 0.8876606822013855),
            ('Practice Work Mathematics-Std.9', 0.9079822301864624),
            ('E3518', 0.9605749249458313),
            ('1', 0.96055477752685547),
            ('PracticeWorkMathematics(Basic)Std.1',
            0.8757174611091614),
            ('E3519', 0.791429877281189),
            ('PracticeWorkMathematics(Standard)Std1',
            0.8527902364730835),
            ],
        'html': '<html><body><table><tr><td>E3509</td><td></td><td>Practice Work Mathematics-Std.9</td></tr><tr><td>E3518</td><td>1</td><td>PracticeWorkMathematics(Basic)Std.1</td></tr><tr><td>E3519</td><td></td><td>PracticeWorkMathematics(Standard)Std1</td></tr></table></body></html>',
        },
    'img_idx': 0,
    }


df, = pd.read_html(data['res']['html'])
df.columns = ['code', 'qty', 'description']

confidences = [
    {
        'detected_object': detected_object,
        'confidence': confidence,
    }
    for detected_object, confidence in data['res']['rec_res']
]
code_confidences = pd.DataFrame([c for c in confidences if c['detected_object'].startswith('E')]).set_index('detected_object')
description_confidences = pd.DataFrame([c for c in confidences if not c['detected_object'].startswith('E')]).set_index('detected_object')


detected_objects = (obj for obj, _ in data['res']['rec_res'])
boxes = [
    {
        'detected_object': obj,
        'coordinates': box,
    }
    for obj, box in zip(detected_objects, data['res']['boxes'])
]

code_boxes = pd.DataFrame([b for b in boxes if b['detected_object'].startswith('E')]).set_index('detected_object')
description_boxes = pd.DataFrame([b for b in boxes if not b['detected_object'].startswith('E')]).set_index('detected_object')


output = (
    df
    .merge(code_confidences, left_on='code', right_index=True)
    .merge(description_confidences, left_on='description', right_index=True)
    .merge(code_boxes, left_on='code', right_index=True)
    .merge(description_boxes, left_on='description', right_index=True)
    .rename(columns=
            {
                'confidence_x': 'code_confidence',
                'confidence_y': 'description_confidence',
                'coordinates_x': 'code_box',
                'coordinates_y': 'description_box',
            }
    )    
)

print(output)

Which prints: