Dropping duplicate items from Scrapy pipeline?

Question

my scrapy crawler collects data from a set of urls, but when I run it again to add new content, the old content is saved to my Mongodb database. Is there a way to check if this item is already found in my Mongodb database(duplicate items have the same title field) and if so, drop it from the pipeline. Also, is it better to delete them from the database after they are saved and if so, how would I implement that in my project.

This is my pipeline:

import logging
import pymongo
from scrapy.exceptions import DropItem


class MongoPipeline(object):

collection_name = 'articles'

def __init__(self, mongo_uri, mongo_db):
    self.mongo_uri = mongo_uri
    self.mongo_db = mongo_db

@classmethod
def from_crawler(cls, crawler):
    ## pull in information from settings.py
    return cls(
        mongo_uri=crawler.settings.get('MONGO_URI'),
        mongo_db=crawler.settings.get('MONGO_DATABASE')
    )

def open_spider(self, spider):
    ## initializing spider
    ## opening db connection
    self.client = pymongo.MongoClient(self.mongo_uri)
    self.db = self.client[self.mongo_db]

def close_spider(self, spider):
    ## clean up when spider is closed
    self.client.close()



def process_item(self, item, spider):
    ## how to handle each post


    bbcDict = {}
    if item['art_content'] != []:
        bbcDict['art_content'] = item['art_content']
        bbcDict['date'] = item['date']
        bbcDict['date_str'] = item['date_str']
        bbcDict['title'] = item['title']
        bbcDict['url'] = item['url']
        self.db[self.collection_name].insert_one(dict(bbcDict))
        return item

   # self.db[self.collection_name].insert(dict(item))
   # logging.debug("Post added to MongoDB")
   # return item

This is my crawler

from datetime import datetime as dt
import scrapy
from ArtScraper.items import ArtscraperItem

class PostSpider(scrapy.Spider):


article = ""
name = 'crawly'
allowed_domains = []

start_urls = ['http://feeds.bbci.co.uk/arabic/rss.xml']



def parse(self, response):
    # get the subreddit from the URL
    #sub = response.url.split('/')[4]

    #Get the title

    # parse thru each of the posts
    #for post in response.css('div.thing'):
    articles = response.xpath('//channel/item')
    for article in articles:
        item = ArtscraperItem()
        print ('hello')
        item['date'] = dt.today()
        item['date_str'] = article.xpath('pubDate/text()').extract_first()
        item['url'] = article.xpath('link/text()').extract_first()
        item['title'] = article.xpath('title/text()').extract_first()
        url = item['url']
        yield scrapy.Request(
            url,
            callback=self.parse_article,
            meta={'item': item},  # carry over our item
        )
        #request = scrapy.Request(url, callback=self.parse_article)
        #request.meta['item'] = item
        #yield request

def parse_article(self, response):
    item = response.meta['item']
    pars = response.xpath("//div[@class='story-body']/div[@class='story-body__inner']/p/text()").extract()
    item['art_content'] = '-'.join(pars)
    print ("HHHH")
    yield item

Thanks in advance.

user_6678896 · Answer

You can filter out duplicates by creating a list of titles on your MongoPipeline class as the items are processed, and use DropItem to delete items during process_items. The official docs provide a great example. You can then save to MongoDB when the item is returned.

In your case here, this would be the implementation of a duplicates filter in your pipeline:

import logging
import pymongo
from scrapy.exceptions import DropItem


class MongoPipeline(object):

collection_name = 'articles'

def __init__(self, mongo_uri, mongo_db):
    self.mongo_uri = mongo_uri
    self.mongo_db = mongo_db
    self.titles_seen = set()

@classmethod
def from_crawler(cls, crawler):
    ## pull in information from settings.py
    return cls(
        mongo_uri=crawler.settings.get('MONGO_URI'),
        mongo_db=crawler.settings.get('MONGO_DATABASE')
    )

def open_spider(self, spider):
    ## initializing spider
    ## opening db connection
    self.client = pymongo.MongoClient(self.mongo_uri)
    self.db = self.client[self.mongo_db]

def close_spider(self, spider):
    ## clean up when spider is closed
    self.client.close()

def process_item(self, item, spider):
    if item['title'] in self.titles_seen:
        raise DropItem("Duplicate item title found: %s" % item)
    else:
        self.titles_seen.add(item['title'])
        return item

user_13225444 · Answer

For me was necessary import the ItemAdapter to convert the Item to Array

from itemadapter import ItemAdapter

def process_item(self, item, spider):
            adapter = ItemAdapter(item)
            if self.db[self.collection_name].find_one({'id':adapter['id']}) != None:
                dado = self.db[self.collection_name].find_one_and_update({'id':adapter['id']})
                ## ----> raise DropItem(f"Duplicate item found: {item!r}") <------
                print(f"Duplicate item found: {dado!r}")
            else:
                self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
            return item

I preferred to update rather than trigger dropitem.

Dropping duplicate items from Scrapy pipeline?

2 Answers