Scrapy: yield argument to non-consecutive callback

user_13592965 0
Viewed 56

I'm new to Scrapy but so far I managed to code this spider (which actually works as intended, almost).

I want to name the downloaded files f'{issue['number']}_{issue['date']}.pdf' but so far I couldn't. name is just a temporary workaround I came up with.

I was looking at Itemloader documentation, maybe that's what I need, but implies rewriting the whole code. Maybe there's a simpler solution. I will go on reading the documentation.

Any tip is welcome, thanks in advance.

PD: English is my second language, and I'm still reading Scrapy documentation ;) If you are wondering why Venezuela, one of my tasks is to import those documents into our database.

ftp_connection = ftplib.FTP(host=ftp_host, user=ftp_user, passwd=ftp_password)
print(ftp_connection.getwelcome())
ftp_connection.cwd(ftp_directory)
ftp_files = ftp_connection.nlst()
print('Successully created list of file names')
ftp_connection.quit()

class DoVenezuela(scrapy.Spider):

    name = 'do_venezuela'
    start_urls = ['http://spgoin.imprentanacional.gob.ve/cgi-win/be_alex.cgi?forma=FGENERAL&nombrebd=spgoin&c01=Titulo&m01=frase&t01=&c03=Descriptor_TGO1&m03=comienzo&c04=FechaInicio&m04=%3E%3D&t04=01-01-2021&c05=FechaInicio&t05=&c06=Descriptor_EDR1&m06=frase&t06=Publicado&TSalida=T%3AGeneralGCTOF&recuperar=3000&MostrarHijos=E&Cizq=2&xsl=&pxsl=&TipoDoc=GCTOF&Submit2=Buscar&Orden=;FID;']

    def parse(self, response):
        even_rows = response.css('tr.LineaTablaImpar')
        odd_rows = response.css('tr.LineaTablaPar')
        all_rows = even_rows + odd_rows

        for row in all_rows:
            issue = Issue()
            issue['number'] = row.css('a.DocTitulo::text').get().replace('.', '')
            issue['edition'] = row.css('a.RefDescriptor::text').get()
            issue['date'] = row.css('td::text')[2].get().replace('-', '')
            issue['link'] =  row.css('a.DocTitulo').attrib['href']

            lookup = f'{issue["number"]}_{issue["date"]}.pdf'

            if lookup in ftp_files:
                print(f'Skipping {lookup}.pdf: already in ftp')
            elif os.path.isfile(f'{download_directory}/{lookup}'):
                print(f'Skipping {lookup}.pdf: already downloaded')
            else:
                yield response.follow(url=f'http://spgoin.imprentanacional.gob.ve{issue["link"]}', callback=self.parse_link1)

    def parse_link1(self, response):
        link1 = response.css('a')[21].attrib['href']
        yield response.follow(url=f'http://spgoin.imprentanacional.gob.ve{link1}', callback=self.parse_link2)

    def parse_link2(self, response):
        link2 = response.css('a')[17].attrib['href']
        yield response.follow(url=f'http://spgoin.imprentanacional.gob.ve{link2}', callback=self.download_pdf)
    
    def download_pdf(self, response):
        name = response.url.split('/')[4].replace('be_alex.cgi?Documento=', '')
        pdf_file = open(f'{download_directory}/{name}.pdf', 'wb')
        pdf_file.write(response.body)
scrapy yield python
1 Answers

I managed to solve this by passing meta keyword all over the code, not sure if this is the most convenient solution... According to the Scrapy docs, cb_kwargs (https://docs.scrapy.org/en/latest/topics/request-response.html#scrapy.http.Request.cb_kwargs) is preferable.

class DoVenezuela(scrapy.Spider):
    name = 'do_venezuela'
    start_urls = ['http://spgoin.imprentanacional.gob.ve/cgi-win/be_alex.cgi?forma=FGENERAL&nombrebd=spgoin&c01=Titulo&m01=frase&t01=&c03=Descriptor_TGO1&m03=comienzo&c04=FechaInicio&m04=%3E%3D&t04=01-01-2021&c05=FechaInicio&t05=&c06=Descriptor_EDR1&m06=frase&t06=Publicado&TSalida=T%3AGeneralGCTOF&recuperar=3000&MostrarHijos=E&Cizq=2&xsl=&pxsl=&TipoDoc=GCTOF&Submit2=Buscar&Orden=;FID;']

    def parse(self, response):
        even_rows = response.css('tr.LineaTablaImpar')
        odd_rows = response.css('tr.LineaTablaPar')
        all_rows = even_rows + odd_rows

        for row in all_rows:
            issue = Issue()
            issue['number'] = row.css('a.DocTitulo::text').get().replace('.', '')
            issue['edition'] = row.css('a.RefDescriptor::text').get()
            issue['date'] = row.css('td::text')[2].get().replace('-', '')
            issue['link'] =  row.css('a.DocTitulo').attrib['href']

            file_name = f'{issue["number"]}_{issue["date"]}.pdf'

            if file_name in ftp_files:
                print(f'Skipping {file_name}.pdf: already in ftp')
            elif os.path.isfile(f'{download_directory}/{file_name}'):
                print(f'Skipping {file_name}.pdf: already downloaded')
            else:

                yield response.follow(
                    url=f'http://spgoin.imprentanacional.gob.ve{issue["link"]}',
                    callback=self.parse_link1,
                    meta={'file_name': file_name}
                )

    def parse_link1(self, response):
        link1 = response.css('a')[21].attrib['href']
        file_name = response.meta.get('file_name')
        yield response.follow(
            url=f'http://spgoin.imprentanacional.gob.ve{link1}',
            callback=self.parse_link2,
            meta={'file_name': file_name}
        )

    def parse_link2(self, response):
        link2 = response.css('a')[17].attrib['href']
        file_name = response.meta.get('file_name')

        yield response.follow(
            url=f'http://spgoin.imprentanacional.gob.ve{link2}',
            callback=self.download_pdf,
            meta={'file_name': file_name}
        )

    def download_pdf(self, response):
        file_name = response.meta.get('file_name')
        pdf_file = open(f'{download_directory}/{file_name}', 'wb')
        pdf_file.write(response.body)

Still working on a better solution.

user_135929650
Related