I want to update __VIEWSTATE and __EVENTVALIDATION values after scraping it from the page.
I am looing over pages and there is a unique __VIEWSTATE and __EVENTVALIDATION values which needs to be referenced in the data dictionary.
I want to know if that's possible to access the response variable outside of class scope so that I can send the value to the data dictionary?
from typing import Counter
import requests
from scrapy.selector import Selector
class Scraper:
val = ''
def bot(self, event_state='some-values', event_validate='some-valus'):
cookies = {}
headers = {}
counter = 0
while True:
data = {
'__EVENTTARGET': f'dgSearchResults$ctl24$ctl0{counter}',
'__VIEWSTATE': self.event_state,
'__VIEWSTATEGENERATOR': '11C1F95B',
'__EVENTVALIDATION': self.event_validate,
'cboType': 'RE',
'DateOfFilingFrom': '01/01/2021',
'DateOfFilingTo': '10/20/2021',
'cboPartyType': 'Decedent',
'cmdSearch': 'Search'
}
url = 'https://registers.maryland.gov/RowNetWeb/Estates/frmEstateSearch2.aspx'
r = requests.post(url=url, headers=headers, cookies=cookies, data=data)
response = Selector(text=r.text)
links = response.xpath('//a[contains(@href, "RecordId")]')
if links == []:
break
for link in links:
ids = link.xpath('.//@href').get()
print(ids)
counter += 1
event_state_value = response.xpath('//input[@name="__VIEWSTATE"]/@value').get()
event_validate_value = response.xpath('//input[@name="__EVENTVALIDATION"]/@value').get()
scraper = Scraper()
scraper.bot(event_state_value, event_validate_value)