I am trying to parse the irregularly structured HTML with scrapy and itemloader, not knowing if an element exists. Is it possible to let the itemloader decide if an element exists and process it to different fields based on the content of the element? I.e. is there a more elegant way of writing the below syntax?
uls = '''
<ul>
<li> 'item W'
<ul>
<li> 'shape: round' </li>
<li> 'color: yellow' </li>
</ul>
</li>
</ul>
<ul>
<li> 'item X'
<ul>
<li> 'green' </li>
<li> 'shape: square' </li>
</ul>
</li>
</ul>
<ul>
<li> 'item Y'
<ul>
<li> 'color: blue' </li>
</ul>
</li>
</ul>
<ul>
<li> 'item Z'
<ul>
<li> 'hexagon' </li>
</ul>
</li>
</ul>
'''
for ul in uls:
loader = ItemLoader(item=Item())
loader.add_xpath('item', './li[1]//text()')
if ul.xpath('./li[1]/ul/li[1]/text()').get() is not None:
text1 = ul.xpath('./li[1]/ul/li[1]/text()').get()
if ul.xpath('./li[1]/ul/li[2]/text()').get() is not None:
text2 = ul.xpath('./li[1]/ul/li[2]/text()').get()
if ul.xpath('./li[1]/ul/li[1]/text()').get() is not None:
if 'shape' in text1:
loader.add_xpath('shape', './li[1]/ul/li[1]/text()')
elif 'color' in text1:
loader.add_xpath('color', './li[1]/ul/li[1]/text()')
elif ul.xpath('./li[1]/ul/li[2]/text()').get() is None:
loader.add_xpath('shape', './li[1]/ul/li[1]/text()')
if ul.xpath('./li[1]/ul/li[2]/text()').get() is not None:
if 'shape' in text2:
loader.add_xpath('shape', './li[1]/ul/li[2]/text()')
elif 'color' in text2:
loader.add_xpath('color', './li[1]/ul/li[2]/text()')