Why modify function in item doesn't work? Scrapy

Viewed 57

I made def convert_date because dates that I scrape from the website look like this 2021-05-06T23:59:59 and also this is the expiration date so I have to subtract 30 days from this to generate the publication date. I'm looking for a reason why my modify function doesn't work. I've used the input and output process I but I still got in the database unchanged string. Could someone give me some hints about this?

#items.py

from scrapy.item import Item, Field
from datetime import timedelta, date
from scrapy.loader.processors import MapCompose, TakeFirst


def convert_date(text):
    text = text.split('T')
    year = int(text[0][:4])
    month = int(text[0][5:7])
    day = int(text[0][8:10])
    process_date = date(year, month, day)
    final_date = process_date - timedelta(30)
    return final_date


class JobOfferItem(Item):
    title = Field()
    employer = Field()
    country = Field()
    employer_page = Field()
    pub_date = Field(
        input_processor=MapCompose(convert_date),
        output_processor=TakeFirst()
        )
    salary = Field()
    seniority = Field()
    offer_id = Field()
    region = Field()
    city = Field()
    url = Field()

#pipilines.py

import sqlite3


class OfferscraperPipeline:
    def __init__(self):
        self.create_connection()
        self.create_table()

    def create_connection(self):
        self.conn = sqlite3.connect('joboffer.db')
        self.curr = self.conn.cursor()

    def create_table(self):
        self.curr.execute("""create table if not exists offer_tb(
                            title text,
                            employer text,
                            country text,
                            employer_page text,
                            pub_date text,
                            salary text,
                            seniority text,
                            offer_id text unique,
                            region text,
                            city text,
                            url text
                            )""")

    def store_db(self, item):
        self.curr.execute("""insert into offer_tb values (?,?,?,?,?,?,?,?,?,?, ?)""", (
            item['title'][0],
            item['employer'][0],
            item['country'][0],
            item['employer_page'][0],
            item['pub_date'][0],
            item['salary'][0],
            item['seniority'][0],
            item['offer_id'][0],
            item['region'][0],
            item['city'][0],
            item['url'],
        ))
        self.conn.commit()

    def process_item(self, item, spider):
        self.store_db(item)
        return item
0 Answers
Related