How can I conditionally get values from XML w/ElementTree?

Viewed 80

I have XML and I need to extract some values, and if these values don't exists I want "N/A".

<?xml version="1.0" encoding="utf-8"?>
<DEF xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="FF.xsd">
    <FOLD SERV="TRYING" VERSION="918" PLATFORM="UNIX" GROUP_NAME="UNIX" MODIFIED="False" LAST_UPLOAD="20220117145134UTC" TYPE="1" USED_BY_CODE="0">
        <JOB JOBID="835" APP="TRY" JOBNAME="JOBA" DESC="Extrac607" TYPE="db" FORM="Databases" CM_VER="N/A" MULTY_AGENT="N" VERSION_SERIAL="1" PARENT_FOLDER="UNIX">
            <VARIABLE NAME="%%TYPE_DB" VALUE="Open Query" />
            <VARIABLE NAME="%%APPLOG" VALUE="N" />
            <VARIABLE NAME="%%ACCOUNT" VALUE="ONLINE" />
            <VARIABLE NAME="%%TYPE" VALUE="Oracle" />
            <VARIABLE NAME="%%VERS" VALUE="11g" />
            <VARIABLE NAME="%%LENGHT" VALUE="16" />
        </JOB>
        <JOB JOBID="839" APP="TRY" JOBNAME="JOBB" DESC="Extrac617" TYPE="db" FORM="Databases" CM_VER="N/A" MULTY_AGENT="N" VERSION_SERIAL="1" PARENT_FOLDER="UNIX">
            <VARIABLE NAME="%%TYPEDB" VALUE="Open Query" />
            <VARIABLE NAME="%%ACCOUNT" VALUE="ONLINE" />
            <VARIABLE NAME="%%TYPE" VALUE="Oracle" />
            <VARIABLE NAME="%%VERS" VALUE="11g" />
            <VARIABLE NAME="%%LENGHT" VALUE="16" />
        </JOB>
    </FOLD>
</DEF>

Here is my code:

from xml.etree import ElementTree
from contextlib import redirect_stdout
from collections import Counter

import xlsxwriter
import pprint
import os
import datetime

begin_time = datetime.datetime.now()
print(datetime.datetime.now())
    
###VARIABLES RUTASL   
fileXML= 'C:\\xxxx\\completa.xml'
cont=0
path= 'C:\\xxxxxx\\todo.csv'
outputExcel = 'C:\\xxxxxxx\\salida.xlsx'
file2 = 'C:\\xxxxxxxx\\todo.csv'




try:
    os.remove(path)
except OSError as e:  ## if failed, report it back to the user ##
    print ("Error: %s - %s." % (e.filename, e.strerror))

try:
    os.remove(outputExcel)
except OSError as e:  ## if failed, report it back to the user ##
    print ("Error: %s - %s." % (e.filename, e.strerror))




with open(fileXML, encoding="utf8") as f:
    tree = ElementTree.parse(f)

    
# using getchildren() within root and check if tag starts with keyword 
#print [node.text for node in root.getchildren() if node.tag.startswith('%%FTP-LPATH')]
root = tree.getroot()
set_tipos= set()
lista_tipos = list()

for node in tree.iter('JOB'):


    name = node.attrib.get('JOBNAME')
    appl_type = node.attrib.get('TYPE')
    appl_form = node.attrib.get('FORM')
    

    if appl_form:
        set_tipos.add(appl_form)
        lista_tipos.append(appl_form)
    else:
        set_tipos.add(appl_type)
        lista_tipos.append(appl_type)
        
     
    
print('#####################################')


cuenta1 = Counter(lista_tipos)
print(cuenta1)

workbook = xlsxwriter.Workbook(outputExcel)
worksheet = workbook.add_worksheet('Resumen')
bold =workbook.add_format({'bold':1})
centrado =workbook.add_format({'center_across':1})
centrado = workbook.add_format({'bold': True, 'center_across': True})

headings = list()
datos = list()
listaBD = list()
pointDB=1

for key, value in cuenta1.items():
    # scores = 0
    print(key)
    headings.append(key)
    datos.append(value)



if 'Databases' in headings:
    print('Existe base de datos')
    
    for node4 in tree.iterfind(".//JOB[@TYPE = 'db']"): 
        listaBD.clear()
        name = node4.attrib.get('JOBNAME')
        appl_type = node4.attrib.get('TYPE')
        listaBD.append(name)
        listaBD.append(appl_type)
        for node4 in tree.iterfind(".//JOB/VARIABLE[@NAME='%%TYPE_DB']"):
            a=0 

        value = node4.attrib.get('VALUE', 'N/A')
        listaBD.append(value) 


        worksheet.write_row('A'+str(pointDB),listaBD)
        pointDB += 1


        node4.clear()

else:
    print('Don't exists')

print(datetime.datetime.now() - begin_time)

workbook.close()

                ...................

This works, but I have 6 if(s) in every case, and the program is very slow. I've edited the main code. It seems problem is the size of xml. XML has 50 MB. The output is: Output How else can I conditionally get these values?

2 Answers

I see at least two things you can change to remove if-blocks.

A more specific X-Path

I see this "if type matches do everything, else skip" branch:

appl_type = node4.attrib.get('TYPE')
if appl_type == 'db':

You can avoid that by putting the check for @TYPE into your top X-Path:

for node4 in tree.iterfind(".//JOB[@TYPE = 'db']"):

Default getter

You're getting an attrib value and deciding what to do based on whether it's None or not:

if node4.attrib.get('VALUE') is None:
    print("don't exists")
    listaBD.append('N/A')
else:
    print("Existe")
    listaBD.append(node4.attrib.get('VALUE'))

Use a default value in the get() method if it's None and avoid the check:

value = node4.attrib.get('VALUE', 'N/A')
listaBD.append(value)

Miscellaneous

This doesn't remove an if-block, but it restructures and makes the code a little easier to follow:

if 'DB' not in headings:
    quit()  # or `return` from a function/method... don't go on

print('DB Exists')

Doesn't remove the if, but now everything is not nested under it.

Here's my complete edit of your original code

from xml.etree import ElementTree as ET

tree = ET.parse('input.xml')
headings = ['DB']
listaBD = []

if 'DB' not in headings:
    quit()  # or `return` from a function/method... don't go on

print('DB Exists')

for node4 in tree.iterfind(".//JOB[@TYPE = 'db']"):
    print('//////////////////////////')
    print(node4.tag, node4.attrib)
    name = node4.attrib.get('JOBNAME')
    appl_type = node4.attrib.get('TYPE')
    
    print(name + " " + appl_type)
    for node4 in tree.iterfind(".//JOB/VARIABLE[@NAME='%%TYPE_DB']"):
        print('TIPO')
        print(node4.attrib.get('VALUE'))

    value = node4.attrib.get('VALUE', 'N/A')
    listaBD.append(value)

    node4.clear()
    for node4 in tree.iterfind(".//JOB/VARIABLE[@NAME='%%DBSCHEMA']"):
        print(node4)

I see you iterating over the XML node-by-node a lot, you even do two complete passes over it, once to build the lists and sets for the headings, and then again to try and find values. I could see that taking some time, but I still don't understand exactly why this is so slow for you.

I suggest you break your problem down into separate steps:

  1. Read over the XML once, building up a simple Python-list of rows with ALL the data... don't try to be selective
  2. Filter the list of rows
  3. Write the filtered list (to your Excel workbook)

How you pull the data out of the XML also can be improved, iterating over every node and checking if the node is what you want is wasteful. Use find('NODE-NAME') to go for exactly what you want.

Here's how I'd do this:

Read the XML once, gather ALL data

from xml.etree import ElementTree as ET

# Pre-define the values you want/expect, note that Type_DB and Account are already 'N/A' (as placeholders)
Row_Tmpl = {'JobID': None, 'JobName': None,
            'Type': None, 'Form': None, 'Type_DB': 'N/A', 'Account': 'N/A'}

# Don't need `with open()`
root = ET.parse('input.xml')

rows = []
for job_node in root.findall('.//JOB'):
    row = dict(Row_Tmpl)  # use dict() to make a copy; don't use it directly

    # Job attributes
    row['JobID'] = job_node.get('JOBID')
    row['JobName'] = job_node.get('JOBNAME')
    row['Type'] = job_node.get('TYPE')
    row['Form'] = job_node.get('FORM')

    # Job's children values
    type_db_node = job_node.find('./VARIABLE[@NAME="%%TYPE_DB"]')
    if type_db_node != None:
        row['Type_DB'] = type_db_node.get('VALUE', 'N/A')

    account_node = job_node.find('./VARIABLE[@NAME="%%ACCOUNT"]')
    if account_node != None:
        row['Account'] = account_node.get('VALUE', 'N/A')

    rows.append(row)

Here's what all the data looks like (I added another JOB to the input XML for my testing):

import pprint

print('All rows from XML')
pprint.pprint(rows, width=180)
All rows from XML
[{'Account': 'ONLINE', 'Form': 'Databases', 'JobID': '835', 'JobName': 'JOBA', 'Type': 'db', 'Type_DB': 'Open Query'},
 {'Account': 'ONLINE', 'Form': 'Databases', 'JobID': '839', 'JobName': 'JOBB', 'Type': 'db', 'Type_DB': 'N/A'},
 {'Account': 'ONLINE', 'Form': 'Databases', 'JobID': '842', 'JobName': 'JOBC', 'Type': 'db', 'Type_DB': 'N/A'}]

Filter w/plain Python

Now, you can filter it, and even modify values however you like, with plain Python:

rows_with_type_db = [x for x in rows if x['Type_DB'] != 'N/A']

print('Rows filtered by Type_DB != N/A')
pprint.pprint(rows_with_type_db, width=180)
Rows filtered by Type_DB != N/A
[{'Account': 'ONLINE', 'Form': 'Databases', 'JobID': '835', 'JobName': 'JOBA', 'Type': 'db', 'Type_DB': 'Open Query'}]

Write it

I don't have the module for creating Excel files, but here's how easy it is to create a CSV with the DictWriter:

from csv import DictWriter

Fieldnames = Row_Tmpl.keys()
with open('filtered_jobs.csv', 'w', newline='') as f:
    writer = DictWriter(f, fieldnames=Fieldnames)
    writer.writeheader()
    writer.writerows(rows_with_type_db)
JobID JobName Type Form Type_DB Account
835 JOBA db Databases Open Query ONLINE
Related