<commit_before>import pymongo
import datetime
import logging
import settings
import uuid
from scrapy import signals


class MongoPipeline(object):
    def __init__(self):
        self.host = '{host}'
        self.port = {port}
        self.user = '{user}'
        self.pwd = '{<PASSWORD>}'
        self.db_data_name = '{db_name}'
        self.db_stat_name = 'crawl_stats'
        self.on_duplicate = '{on_dup}'
        self.collection_name = '{coll_name}'
        self.keys = [{keys}]
        self.client = pymongo.MongoClient(self.host, self.port)

    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls()
        crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
        return pipeline

    def open_spider(self, spider):
        try: self.job_id = spider._job
        except AttributeError: self.job_id = str(uuid.uuid4())
        self._init_stats(spider)
        self._store_stats(spider)

    def process_item(self, item, spider):
        item = self._convert_keys_to_string(dict(item))
        item['crawl_datetime'] = datetime.datetime.now()
        self._store_item(item, spider)
        return item

    def spider_closed(self, spider):
        self._store_stats(spider)
        self.client.close()

    def _store_item(self, item, spider):
        key_filter = self._get_key_filter(item)
        self._select_data_db()
        if self.db[self.collection_name].find(key_filter).count() > 0:
            if self.on_duplicate == 'stop':
                spider.crawler.engine.close_spider(spider, 'Duplicated item found.')
            elif self.on_duplicate == 'update':
                self.db[self.collection_name].update(key_filter, {{'$set': item}}, multi=True)
                spider.crawler.stats.inc_value('update_item_count')
            elif self.on_duplicate == 'ignore':
                pass
            else:
                raise ValueError('Invalid on duplicate action.')
        else:
            self.db[self.collection_name].insert(item)
            spider.crawler.stats.inc_value('insert_item_count')
            self._store_stats(spider)

    def _store_stats(self, spider):
        stats = spider.crawler.stats.get_stats()
        stats['_id'] = self.job_id
        stats['project'] = settings.BOT_NAME
        stats['spider'] = spider.name
        self._select_stat_db()
        self.db['crawl_stats'].find_and_modify(query={{'_id': self.job_id}}, update=stats, upsert=True)

    def _select_data_db(self):
        self.db = self.client[self.db_data_name]

    def _select_stat_db(self):
        self.db = self.client[self.db_stat_name]

    def _init_stats(self, spider):
        spider.crawler.stats.set_value('update_item_count', 0)
        spider.crawler.stats.set_value('insert_item_count', 0)

    def _get_key_filter(self, item):
        key_filter = dict()
        for key in self.keys:
            key_filter[key] = item[key]
        return key_filter

    def _convert_keys_to_string(self, d):
        r = {{}}
        for key in d:
            if type(d[key]) is dict:
                r[str(key)] = self._convert_keys_to_string(d[key])
            else:
                r[str(key)] = d[key]
        return r

<commit_msg>Change the uuid generation to be the same with scrapyd.<commit_after>import pymongo
import datetime
import logging
import settings
import uuid
from scrapy import signals


class MongoPipeline(object):
    def __init__(self):
        self.host = '{host}'
        self.port = {port}
        self.user = '{user}'
        self.pwd = <PASSWORD>}'
        self.db_data_name = '{db_name}'
        self.db_stat_name = 'crawl_stats'
        self.on_duplicate = '{on_dup}'
        self.collection_name = '{coll_name}'
        self.keys = [{keys}]
        self.client = pymongo.MongoClient(self.host, self.port)

    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls()
        crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
        return pipeline

    def open_spider(self, spider):
        try: self.job_id = spider._job
        except AttributeError: self.job_id = uuid.uuid1().hex
        self._init_stats(spider)
        self._store_stats(spider)

    def process_item(self, item, spider):
        item = self._convert_keys_to_string(dict(item))
        item['crawl_datetime'] = datetime.datetime.now()
        self._store_item(item, spider)
        return item

    def spider_closed(self, spider):
        self._store_stats(spider)
        self.client.close()

    def _store_item(self, item, spider):
        key_filter = self._get_key_filter(item)
        self._select_data_db()
        if self.db[self.collection_name].find(key_filter).count() > 0:
            if self.on_duplicate == 'stop':
                spider.crawler.engine.close_spider(spider, 'Duplicated item found.')
            elif self.on_duplicate == 'update':
                self.db[self.collection_name].update(key_filter, {{'$set': item}}, multi=True)
                spider.crawler.stats.inc_value('update_item_count')
            elif self.on_duplicate == 'ignore':
                pass
            else:
                raise ValueError('Invalid on duplicate action.')
        else:
            self.db[self.collection_name].insert(item)
            spider.crawler.stats.inc_value('insert_item_count')
            self._store_stats(spider)

    def _store_stats(self, spider):
        stats = spider.crawler.stats.get_stats()
        stats['_id'] = self.job_id
        stats['project'] = settings.BOT_NAME
        stats['spider'] = spider.name
        self._select_stat_db()
        self.db['crawl_stats'].find_and_modify(query={{'_id': self.job_id}}, update=stats, upsert=True)

    def _select_data_db(self):
        self.db = self.client[self.db_data_name]

    def _select_stat_db(self):
        self.db = self.client[self.db_stat_name]

    def _init_stats(self, spider):
        spider.crawler.stats.set_value('update_item_count', 0)
        spider.crawler.stats.set_value('insert_item_count', 0)

    def _get_key_filter(self, item):
        key_filter = dict()
        for key in self.keys:
            key_filter[key] = item[key]
        return key_filter

    def _convert_keys_to_string(self, d):
        r = {{}}
        for key in d:
            if type(d[key]) is dict:
                r[str(key)] = self._convert_keys_to_string(d[key])
            else:
                r[str(key)] = d[key]
        return r


{'fraction_non_alphanumeric': 0.1170262300170728, 'fraction_numerical': 0.0012416576129132392, 'mean_word_length': 2.424017003188098, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '515083', 'n_tokens_mistral': 2089, 'n_tokens_neox': 2030, 'n_words': 369}