Create dataframe from AWS textract form kv output where keys are column headers and values are in single row

Viewed 9

I am using AWS Textract to analyze a form document, the output is KV pairs. I am trying to create a dataframe using the Key as the column header and the values in a single row then use some of those values and insert them into my database. I am new to coding and this is my first post so please take it easy on me.

import boto3
import sys
import re
import json
from collections import defaultdict


def get_kv_map(file_name):
    with open(file_name, 'rb') as file:
        img_test = file.read()
        bytes_test = bytearray(img_test)
        print('Image loaded', file_name)

    # process using image bytes
    client = boto3.client('textract')
    response = client.analyze_document(Document={'Bytes': bytes_test}, FeatureTypes=['FORMS'])

    # Get the text blocks
    blocks = response['Blocks']

    # get key and value maps
    key_map = {}
    value_map = {}
    block_map = {}
    for block in blocks:
        block_id = block['Id']
        block_map[block_id] = block
        if block['BlockType'] == "KEY_VALUE_SET":
            if 'KEY' in block['EntityTypes']:
                key_map[block_id] = block
            else:
                value_map[block_id] = block

    return key_map, value_map, block_map


def get_kv_relationship(key_map, value_map, block_map):
    kvs = defaultdict(list)
    for block_id, key_block in key_map.items():
        value_block = find_value_block(key_block, value_map)
        key = get_text(key_block, block_map)
        val = get_text(value_block, block_map)
        kvs[key].append(val)
    return kvs


def find_value_block(key_block, value_map):
    for relationship in key_block['Relationships']:
        if relationship['Type'] == 'VALUE':
            for value_id in relationship['Ids']:
                value_block = value_map[value_id]
    return value_block


def get_text(result, blocks_map):
    text = ''
    if 'Relationships' in result:
        for relationship in result['Relationships']:
            if relationship['Type'] == 'CHILD':
                for child_id in relationship['Ids']:
                    word = blocks_map[child_id]
                    if word['BlockType'] == 'WORD':
                        text += word['Text'] + ' '
                    if word['BlockType'] == 'SELECTION_ELEMENT':
                        if word['SelectionStatus'] == 'SELECTED':
                            text += 'X '

    return text


def print_kvs(kvs):
    for key, value in kvs.items():
        print(key, ":", value)


def search_value(kvs, search_key):
    for key, value in kvs.items():
        if re.search(search_key, key, re.IGNORECASE):
            return value


def main(file_name):
    key_map, value_map, block_map = get_kv_map(file_name)

    # Get Key Value relationship
    kvs = get_kv_relationship(key_map, value_map, block_map)
    print("\n\n== FOUND KEY : VALUE pairs ===\n")
    print_kvs(kvs)




if __name__ == "__main__":
    file_name = sys.argv[1]
    main(file_name)

OUTPUT:

== FOUND KEY : VALUE pairs ===

    Social#:  : ['XXXX ']
    Landlord Name:  : ['XXXX Trust ']
    Date:  : ['']
    Last Name:  : ['XXX ']
    Date of Birth:  : ['04/30/1959 ', '05/31/1955 ']
    State:  : ['CA ', 'CA ', 'CA ']
    Zip Code:  : ['92860 ', '92860 ', '92860 ']
    City:  : ['Norco ', 'Norco ', 'Norco ']
    Phone #:  : ['XXXX ', 'XXXX ', 'XXX ']
    Landlord Contact#:  : ['XXXX']
    Social Number/SIN#:  : ['XXXX ']
    Signature:  : ['']
    Primary Contact Number:  : ['XXXX ']
    Street Address:  : ['684 XXXX ']
    Physical Location Phone #:  : ['XXXXXXX ']
    Business Physical Street Address:  : ['684 XXXX ']
    Business Start Date of Current Ownership:  : ['05/01/2022 ']
    Amount Requested:  : ['550000.00 ']
    Home Address:  : ['1400 EXXXXX, 2201, Long Beach, CA 90802 ']
    Yes  : ['', 'X ', '', '']
    Industry Type: (Describe)  : ['manufacturing ']
    Business Legal Name:  : ['XXXXXX, LLC ']
    Business Trade Reference #3:  : ['XXXX ']
    Owner First Name  : ['Jim ']
    Billing Location Phone #:  : ['XXXX ']
    Est. Credit Score:  : ['700 ']
    LLC  : ['X ']
    Billing Street Address:  : ['684 XXXXX ']
    No  : ['', 'X ', 'X ', 'X ']
    Rented  : ['X ']
    Use of Proceeds:  : ['XXXXX ']
    Business Tax ID:  : ['XXXX ']
    Agent Name:  : ['']
    Date (M/D/YY)  : ['Sep 01 2022 ']
    Fax Number #:  : ['']
    Current Estimated MCA/ LINE OF CREDIT Balance  : ['350000.00 ']
    Credit Card Processing?  : ['X Yes No ']
    Business Trade Reference #1  : ['XXXXX']
    Business Website Address:  : ['']
    Mortgaged  : ['']
    Gross Annual Sales (from previous year's Tax return):  : ['2900000.00 ']
    Owner / Officer's Signature:  : ['']
    Business DBA Name:  : ['same ']
    Business Trade Reference #2:  : ['UPS ']
    State of Incorporation  : ['XXXX ']
    Ownership Percentage %  : ['37 ']
    Monthly Payment:  : ['7745.00 ']
    Name of Credit Card Processor  : ['XXXX']
    Owner / Officer's Name: (Print)  : ['XXXX ']
    LLP  : ['']
    Second Owner:  : ['XXXX ']
    Merchant Email Address  : ['XXXXX ']
    Fax:  : ['XXXXX ']
0 Answers
Related