<commit_before># -*- coding: utf-8 -*-

import scrapy


class DnaIndiaDebugSpider(scrapy.Spider):
    name = "dnaindia_debug"
    allowed_domains = ["www.dnaindia.com"]
    start_urls = [
        # Pagination
        'http://www.dnaindia.com/analysis',
        # Detail
        # 'http://www.dnaindia.com/analysis/editorial-dnaedit-modi-s-manoeuvre-2231861'
    ]

    # 'Ignoring response <403 http://www.dnaindia.com/analysis>: HTTP status code is not handled or not allowed'

    def __init__(self, name=None, **kwargs):
        from cwpoliticl.database_factory import DatabaseFactory, CollectionTypes
        database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
                                           kwargs['user'], kwargs['passwd'],
                                           kwargs['db'], kwargs['collection_name'])

        self._cache_db = database_factory.get_database(CollectionTypes.cache)
        self._history_db = database_factory.get_database(CollectionTypes.history)
        self._item_db = database_factory.get_database(CollectionTypes.item)

        from cwpoliticl.extensions.dnaindia_parser import DnaIndiaParser
        self._dna_india_Parse = DnaIndiaParser()

        super(DnaIndiaDebugSpider, self).__init__(name, **kwargs)

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        return super(DnaIndiaDebugSpider, cls).from_crawler(crawler,
                                                            args,<commit_msg>cwpoliticl.<commit_after># -*- coding: utf-8 -*-

import scrapy


class DnaIndiaDebugSpider(scrapy.Spider):
    name = "dnaindia_debug"
    allowed_domains = ["www.dnaindia.com"]
    start_urls = [
        # Pagination
        # 'http://www.dnaindia.com/analysis',
        # Detail
        'http://www.dnaindia.com/analysis/editorial-dnaedit-modi-s-manoeuvre-2231861'
    ]

    # 'Ignoring response <403 http://www.dnaindia.com/analysis>: HTTP status code is not handled or not allowed'

    def __init__(self, name=None, **kwargs):
        from cwpoliticl.database_factory import DatabaseFactory, CollectionTypes
        database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
                                           kwargs['user'], kwargs['passwd'],
                                           kwargs['db'], kwargs['collection_name'])

        self._cache_db = database_factory.get_database(CollectionTypes.cache)
        self._history_db = database_factory.get_database(CollectionTypes.history)
        self._item_db = database_factory.get_database(CollectionTypes.item)

        from cwpoliticl.extensions.dnaindia_parser import DnaIndiaParser
        self._dna_india_Parse = DnaIndiaParser()

        super(DnaIndiaDebugSpider, self).__init__(name, **kwargs)

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        return super(DnaIndiaDebugSpider, cls).from_crawler(crawler,
                                                            args,
{'fraction_non_alphanumeric': 0.10747978436657682, 'fraction_numerical': 0.007412398921832884, 'mean_word_length': 2.526128266033254, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6642888', 'n_tokens_mistral': 876, 'n_tokens_neox': 838, 'n_words': 139}