class DuplicatesPipeline(object):

    def __init__(self):
        self.ids_seen = set()

def process_item(self, item, spider):
    if item['projname'] in self.ids_seen:
        pass
Using middleware to prevent scrapy from double-visiting websites