<commit_before>import json

import pymongo
from scrapy.exceptions import DropItem

from bo.items import BoPackagedItem
from bo.utils import preconditions
from bo.utils.exceptions import BoSettingsError

MONGO_DATABASE = 'MONGO_DATABASE'
MONGO_URI = 'MONGO_URI'
MONGO_COLLECTION_NAME = 'MONGO_COLLECTION_NAME'
OUTPUT_FILE = 'OUTPUT_FILE'


class PackagingPipeline(object):
    def process_item(self, bo_pipeline_item, spider):
        packaged_item = BoPackagedItem()
        packaged_item['url'] = bo_pipeline_item.get_url()
        packaged_item['language'] = bo_pipeline_item['sentiment_nlp_result']['language']
        packaged_item['category'] = bo_pipeline_item['category_nlp_result']['category']
        packaged_item['doc_sentiment'] = self.cleanup_doc_sentiment(
                bo_pipeline_item['sentiment_nlp_result']['docSentiment'])
        packaged_item['tags'] = bo_pipeline_item['tags']

        return packaged_item

    @staticmethod
    def cleanup_doc_sentiment(doc_sentiment):
        return {
            'mixed': doc_sentiment['mixed'] == 1,
            'score': float(doc_sentiment['score']),
            'type': doc_sentiment['type']
        }


class JsonFileWriterStage(object):
    def __init__(self, output_file):
        self.file_name = output_file if output_file is not None else 'default_output.debug.json'
        self.file = None

    @classmethod
    def from_crawler(cls, crawler):
        output_file = crawler.settings.get(OUTPUT_FILE)
        return cls(output_file)

    def open_spider(self, spider):
        self.file = open(self.file_name, 'w')

    def close_spider(self, spider):
        self.file.close()

    def process_item(self, packaged_item, spider):
        line = json.dumps(dict(packaged_item), indent=4) + "\n"
        self.file.write(line)
        return packaged_item


class MongoStorageStage(object):
    def __init__(self, mongo_uri, mongo_db, collection_name):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db
        self.collection_name = collection_name
        self.client = None
        self.db = None

    @classmethod
    def from_crawler(cls, crawler):
        mongo_uri = preconditions.check_not_none_or_whitespace(crawler.settings.get(MONGO_URI), MONGO_URI,
                                                               exception=BoSettingsError)
        mongo_db = preconditions.check_not_none_or_whitespace(crawler.settings.get(MONGO_DATABASE), MONGO_DATABASE,
                                                              exception=BoSettingsError)
        collection_name = preconditions.check_not_none_or_whitespace(crawler.settings.get(MONGO_COLLECTION_NAME),
                                                                     MONGO_COLLECTION_NAME, exception=BoSettingsError)

        return cls(mongo_uri=mongo_uri, mongo_db=mongo_db, collection_name=collection_name)

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        post_id = self.db[self.collection_name].insert(dict(item))
        raise DropItem(
                "URL '{0}' successfully crawled and info stored in MongoDB with post ID '{1}'".format(item['url'],
                                                                                                      post_id))<commit_msg>Adding check for existing documents in MongoDB for the same URL<commit_after>import json
import logging

import pymongo
from scrapy.exceptions import DropItem

from bo.items import BoPackagedItem
from bo.utils import preconditions
from bo.utils.exceptions import BoSettingsError

MONGO_DATABASE = 'MONGO_DATABASE'
MONGO_URI = 'MONGO_URI'
MONGO_COLLECTION_NAME = 'MONGO_COLLECTION_NAME'
OUTPUT_FILE = 'OUTPUT_FILE'


class PackagingPipeline(object):
    def process_item(self, bo_pipeline_item, spider):
        packaged_item = BoPackagedItem()
        packaged_item['url'] = bo_pipeline_item.get_url()
        packaged_item['language'] = bo_pipeline_item['sentiment_nlp_result']['language']
        packaged_item['category'] = bo_pipeline_item['category_nlp_result']['category']
        packaged_item['doc_sentiment'] = self.cleanup_doc_sentiment(
                bo_pipeline_item['sentiment_nlp_result']['docSentiment'])
        packaged_item['tags'] = bo_pipeline_item['tags']

        return packaged_item

    @staticmethod
    def cleanup_doc_sentiment(doc_sentiment):
        return {
            'mixed': doc_sentiment['mixed'] == 1,
            'score': float(doc_sentiment['score']),
            'type': doc_sentiment['type']
        }


class JsonFileWriterStage(object):
    def __init__(self, output_file):
        self.file_name = output_file if output_file is not None else 'default_output.debug.json'
        self.file = None

    @classmethod
    def from_crawler(cls, crawler):
        output_file = crawler.settings.get(OUTPUT_FILE)
        return cls(output_file)

    def open_spider(self, spider):
        self.file = open(self.file_name, 'w')

    def close_spider(self, spider):
        self.file.close()

    def process_item(self, packaged_item, spider):
        line = json.dumps(dict(packaged_item), indent=4) + "\n"
        self.file.write(line)
        return packaged_item


class MongoStorageStage(object):
    def __init__(self, mongo_uri, mongo_db, collection_name):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db
        self.collection_name = collection_name
        self.client = None
        self.db = None

    @classmethod
    def from_crawler(cls, crawler):
        mongo_uri = preconditions.check_not_none_or_whitespace(crawler.settings.get(MONGO_URI), MONGO_URI,
                                                               exception=BoSettingsError)
        mongo_db = preconditions.check_not_none_or_whitespace(crawler.settings.get(MONGO_DATABASE), MONGO_DATABASE,
                                                              exception=BoSettingsError)
        collection_name = preconditions.check_not_none_or_whitespace(crawler.settings.get(MONGO_COLLECTION_NAME),
                                                                     MONGO_COLLECTION_NAME, exception=BoSettingsError)

        return cls(mongo_uri=mongo_uri, mongo_db=mongo_db, collection_name=collection_name)

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db][self.collection_name]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        collision = self.db.find_one({'url': item['url']})
        if collision is None:
            post_id = self.db.insert(dict(item))
        else:
            self.db.replace_one({'url': item['url']}, dict(item))
            post_id = collision['_id']
            logging.info(
                "URL: '{0}' was already stored in MongoDB at ID: '{1}'. Will be replaced.".format(item['url'], post_id))

        raise DropItem(
                "URL '{0}' successfully crawled and info stored in MongoDB with post ID '{1}'".format(item['url'],
                                                                                                      post_id))
{'fraction_non_alphanumeric': 0.0943009430094301, 'fraction_numerical': 0.0013666803334700013, 'mean_word_length': 2.5978367748279254, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '109920', 'n_tokens_mistral': 2206, 'n_tokens_neox': 2129, 'n_words': 389}