<commit_before># -*- coding: utf-8 -*-

import logging
import datetime
import dateutil
import html2text
import feedparser

from constance import config
from readability import ParserClient

try:
    from boilerpipe.extract import Extractor as BoilerPipeExtractor

except ImportError:
    BoilerPipeExtractor = None # NOQA

from celery.contrib.methods import task as celery_task_method
from celery.exceptions import SoftTimeLimitExceeded

from pymongo.errors import DuplicateKeyError

from mongoengine import Document, ValidationError
from mongoengine.errors import NotUniqueError
from mongoengine.fields import (IntField, FloatField, BooleanField,
                                DateTimeField,
                                ListField, StringField,
                                URLField,
                                ReferenceField, GenericReferenceField,
                                EmbeddedDocumentField)

from django.conf import settings
from django.core.mail import mail_admins
from django.utils.translation import ugettext_lazy as _
from django.utils.text import slugify

from ...base.utils import (connect_mongoengine_signals,
                           SimpleCacheLock,
                           HttpResponseLogProcessor, RedisStatsCounter)
from .keyval import FeedbackDocument

LOGGER = logging.getLogger(__name__)

feedparser.USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0' # NOQA

CONTENT_NOT_PARSED    = None  # Don't use any _() (fails) nor any lang-dependant values.  # NOQA
CONTENT_TYPE_NONE     = 0
CONTENT_TYPE_HTML     = 1
CONTENT_TYPE_MARKDOWN = 2

now       = datetime.datetime.now
#today     = datetime.date.today
timedelta = datetime.timedelta


# ••••••••••••• issue https://code.google.com/p/feedparser/issues/detail?id=404

def dateutilDateHandler(aDateString):
    return dateutil.parser.parse(aDateString).utctimetuple()

feedparser.registerDateHandler(dateutilDateHandler)

# ••••••••• end issue •••••••••••••••••••••••••••••••••••••••••••••••••••••••••


class FeedStatsCounter(RedisStatsCounter):
    """ This counter represents a given feed's statistics.

        It will increment global feeds statistics too, so that we
        developpers don't have to worry about 2 types of statistics
        counters in the Feed methods / tasks.
    """

    key_base = 'feeds'

    def incr_fetched(self):
        if self.key_base != global_feed_stats.key_base:
            global_feed_stats.incr_fetched()

        return self.fetched(increment=True)

    def fetched(self, increment=False):

        if self.key_base != global_feed_stats.key_base:
            # in case we want to reset.
            global_feed_stats.fetched(increment)

        return RedisStatsCounter._int_incr_key(
            self.key_base + ':fetch', increment)

    def incr_dupes(self):
        if self.key_base != global_feed_stats.key_base:
            global_feed_stats.incr_dupes()

        return self.dupes(increment=True)

    def dupes(self, increment=False):

        if self.key_base != global_feed_stats.key_base:
            # in case we want to reset.
            global_feed_stats.dupes(increment)

        return RedisStatsCounter._int_incr_key(
            self.key_base + ':dupes', increment)

# This one will keep track of all counters, globally, for all feeds.
global_feed_stats = FeedStatsCounter()


class Source(Document):
    """ The "original source" for similar articles: they have different authors,
        different contents, but all refer to the same information, which can
        come from the same article on the net (or radio, etc).

        Eg:
            - article1 on Le Figaro
            - article2 on Liberation
            - both refer to the same AFP news, but have different content.

    """
    type    = StringField()
    uri     = URLField(unique=True)
    name    = StringField()
    authors = ListField(ReferenceField('User'))
    slug    = StringField()


class Feed(Document):
    # TODO: init
    name           = StringField()
    url            = URLField(unique=True)
    site_url       = URLField()
    slug           = StringField()
    restricted     = BooleanField(default=False)
    closed         = BooleanField(default=False)

    fetch_interval = IntField(default=config.FETCH_DEFAULT_INTERVAL)
    last_fetch     = DateTimeField()

    # Stored directly from feedparser data to avoid wasting BW.
    last_etag      = StringField()
    last_modified  = StringField()

    def __unicode__(self):
        return _(u'%s (#%s) from %s') % (self.name, self.id, self.url)

    @classmethod
    def signal_post_save_handler(cls, sender, document, **kwargs):

        # Update the feed with current content.
        document.refresh.delay()

    def get_articles(self, limit=None):

        if limit:
            return Article.objects.filter(
                feed=self).order_by('-date_published').limit(limit)

        return Article.objects.filter(
            feed=self).order_by('-date_published')

    def validate(self, *args, **kwargs):
        try:<commit_msg>add date_added field <commit_after># -*- coding: utf-8 -*-

import logging
import datetime
import dateutil
import html2text
import feedparser

from constance import config
from readability import ParserClient

try:
    from boilerpipe.extract import Extractor as BoilerPipeExtractor

except ImportError:
    BoilerPipeExtractor = None # NOQA

from celery.contrib.methods import task as celery_task_method
from celery.exceptions import SoftTimeLimitExceeded

from pymongo.errors import DuplicateKeyError

from mongoengine import Document, ValidationError
from mongoengine.errors import NotUniqueError
from mongoengine.fields import (IntField, FloatField, BooleanField,
                                DateTimeField,
                                ListField, StringField,
                                URLField,
                                ReferenceField, GenericReferenceField,
                                EmbeddedDocumentField)

from django.conf import settings
from django.core.mail import mail_admins
from django.utils.translation import ugettext_lazy as _
from django.utils.text import slugify

from ...base.utils import (connect_mongoengine_signals,
                           SimpleCacheLock,
                           HttpResponseLogProcessor, RedisStatsCounter)
from .keyval import FeedbackDocument

LOGGER = logging.getLogger(__name__)

feedparser.USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0' # NOQA

CONTENT_NOT_PARSED    = None  # Don't use any _() (fails) nor any lang-dependant values.  # NOQA
CONTENT_TYPE_NONE     = 0
CONTENT_TYPE_HTML     = 1
CONTENT_TYPE_MARKDOWN = 2

now       = datetime.datetime.now
#today     = datetime.date.today
timedelta = datetime.timedelta


# ••••••••••••• issue https://code.google.com/p/feedparser/issues/detail?id=404

def dateutilDateHandler(aDateString):
    return dateutil.parser.parse(aDateString).utctimetuple()

feedparser.registerDateHandler(dateutilDateHandler)

# ••••••••• end issue •••••••••••••••••••••••••••••••••••••••••••••••••••••••••


class FeedStatsCounter(RedisStatsCounter):
    """ This counter represents a given feed's statistics.

        It will increment global feeds statistics too, so that we
        developpers don't have to worry about 2 types of statistics
        counters in the Feed methods / tasks.
    """

    key_base = 'feeds'

    def incr_fetched(self):
        if self.key_base != global_feed_stats.key_base:
            global_feed_stats.incr_fetched()

        return self.fetched(increment=True)

    def fetched(self, increment=False):

        if self.key_base != global_feed_stats.key_base:
            # in case we want to reset.
            global_feed_stats.fetched(increment)

        return RedisStatsCounter._int_incr_key(
            self.key_base + ':fetch', increment)

    def incr_dupes(self):
        if self.key_base != global_feed_stats.key_base:
            global_feed_stats.incr_dupes()

        return self.dupes(increment=True)

    def dupes(self, increment=False):

        if self.key_base != global_feed_stats.key_base:
            # in case we want to reset.
            global_feed_stats.dupes(increment)

        return RedisStatsCounter._int_incr_key(
            self.key_base + ':dupes', increment)

# This one will keep track of all counters, globally, for all feeds.
global_feed_stats = FeedStatsCounter()


class Source(Document):
    """ The "original source" for similar articles: they have different authors,
        different contents, but all refer to the same information, which can
        come from the same article on the net (or radio, etc).

        Eg:
            - article1 on Le Figaro
            - article2 on Liberation
            - both refer to the same AFP news, but have different content.

    """
    type    = StringField()
    uri     = URLField(unique=True)
    name    = StringField()
    authors = ListField(ReferenceField('User'))
    slug    = StringField()


class Feed(Document):
    # TODO: init
    name           = StringField(verbose_name=_(u'name'))
    url            = URLField(unique=True)
    site_url       = URLField()
    slug           = StringField()
    restricted     = BooleanField(default=False)
    closed         = BooleanField(default=False)
    date_added = DateTimeField(default=now)  # added in 1flow database

    fetch_interval = IntField(default=config.FETCH_DEFAULT_INTERVAL)
    last_fetch     = DateTimeField()

    # Stored directly from feedparser data to avoid wasting BW.
    last_etag      = StringField()
    last_modified  = StringField()

    def __unicode__(self):
        return _(u'%s (#%s) from %s') % (self.name, self.id, self.url)

    @classmethod
    def signal_post_save_handler(cls, sender, document, **kwargs):

        # Update the feed with current content.
        document.refresh.delay()

    def get_articles(self, limit=None):

        if limit:
            return Article.objects.filter(
                feed=self).order_by('-date_published').limit(limit)

        return Article.objects.filter(
            feed=self).order_by('-date_published')

    def validate(self, *args, **kwargs):
        try:
{'fraction_non_alphanumeric': 0.0917234664070107, 'fraction_numerical': 0.005939629990262901, 'mean_word_length': 2.741712204007286, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '900200', 'n_tokens_mistral': 3129, 'n_tokens_neox': 2953, 'n_words': 776}