<commit_before>import time
import settings
import difflib
import datetime
import hashlib
import random
import re
from collections import defaultdict
from BeautifulSoup import BeautifulStoneSoup
from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder, TrigramAssocMeasures, BigramAssocMeasures
from django.db import models
from django.db import IntegrityError
from django.core.cache import cache
from utils import json
from utils.feed_functions import levenshtein_distance
from utils.story_functions import format_story_link_date__short
from utils.story_functions import format_story_link_date__long
from utils.story_functions import pre_process_story
from utils.compressed_textfield import CompressedTextField, StoryField
from utils.diff import HTMLDiff

USER_AGENT = 'NewsBlur v1.0 - newsblur.com'

ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)

class Feed(models.Model):
    feed_address = models.URLField(max_length=255, verify_exists=True, unique=True)
    feed_link = models.URLField(max_length=1000, default="", blank=True, null=True)
    feed_title = models.CharField(max_length=255, default="", blank=True, null=True)
    feed_tagline = models.CharField(max_length=1024, default="", blank=True, null=True)
    active = models.BooleanField(default=True)
    num_subscribers = models.IntegerField(default=0)
    last_update = models.DateTimeField(auto_now=True)
    fetched_once = models.BooleanField(default=False)
    has_exception = models.BooleanField(default=False)
    min_to_decay = models.IntegerField(default=15)
    days_to_trim = models.IntegerField(default=90)
    creation = models.DateField(auto_now_add=True)
    etag = models.CharField(max_length=50, blank=True, null=True)
    last_modified = models.DateTimeField(null=True, blank=True)
    stories_last_month = models.IntegerField(default=0)
    average_stories_per_month = models.IntegerField(default=0)
    story_count_history = models.TextField(blank=True, null=True)
    next_scheduled_update = models.DateTimeField(default=datetime.datetime.now)
    last_load_time = models.IntegerField(default=0)
    popular_tags = models.CharField(max_length=1024, blank=True, null=True)
    popular_authors = models.CharField(max_length=2048, blank=True, null=True)
    
    
    def __unicode__(self):
        return self.feed_title

    def save(self, lock=None, *args, **kwargs):
        if self.feed_tagline and len(self.feed_tagline) > 1024:
            self.feed_tagline = self.feed_tagline[:1024]
            
        if lock:
            lock.acquire()
            try:
                super(Feed, self).save(*args, **kwargs)
            finally:
                lock.release()
        else:
            super(Feed, self).save(*args, **kwargs)
        
    def save_feed_history(self, status_code, message, exception=None):
        FeedFetchHistory.objects.create(feed=self, 
                                        status_code=status_code,
                                        message=message,
                                        exception=exception)
        old_fetch_histories = self.feed_fetch_history.all()[10:]
        for history in old_fetch_histories:
            history.delete()
            
        if status_code >= 400:
            fetch_history = self.feed_fetch_history.all().values('status_code')
            self.count_errors_in_history(fetch_history)
        
    def save_page_history(self, status_code, message, exception=None):
        PageFetchHistory.objects.create(feed=self, 
                                        status_code=status_code,
                                        message=message,
                                        exception=exception)
        old_fetch_histories = self.page_fetch_history.all()[10:]
        for history in old_fetch_histories:
            history.delete()
            
        if status_code >= 400:
            fetch_history = self.page_fetch_history.all().values('status_code')
            self.count_errors_in_history(fetch_history)
        
    def count_errors_in_history(self, fetch_history):
        non_errors = [h for h in fetch_history if h['status_code'] < 400]
        errors = [h for h in fetch_history if h['status_code'] >= 400]

        if len(non_errors) == 0 and len(errors) >= 1:
            self.has_exception = True
            self.active = False
            self.save()
    
    def count_subscribers(self, verbose=False, lock=None):
        from apps.reader.models import UserSubscription
        subs = UserSubscription.objects.filter(feed=self)
        self.num_subscribers = subs.count()

        self.save(lock=lock)
        
        if verbose:
            if self.num_subscribers <= 1:
                print '.',
            else:
                print "\n %s> %s subscriber%s: %s" % (
                    '-' * min(self.num_subscribers, 20),
                    self.num_subscribers,
                    '' if self.num_subscribers == 1 else 's',
                    self.feed_title,
                ),

    def count_stories(self, verbose=False, lock=None):
        month_ago = datetime.datetime.now() - datetime.timedelta(days=30)
        stories_last_month = Story.objects.filter(story_feed=self, story_date__gte=month_ago).count()
        self.stories_last_month = stories_last_month
        <commit_msg>Coercing errors into integers. Doh.<commit_after>import time
import settings
import difflib
import datetime
import hashlib
import random
import re
from collections import defaultdict
from BeautifulSoup import BeautifulStoneSoup
from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder, TrigramAssocMeasures, BigramAssocMeasures
from django.db import models
from django.db import IntegrityError
from django.core.cache import cache
from utils import json
from utils.feed_functions import levenshtein_distance
from utils.story_functions import format_story_link_date__short
from utils.story_functions import format_story_link_date__long
from utils.story_functions import pre_process_story
from utils.compressed_textfield import CompressedTextField, StoryField
from utils.diff import HTMLDiff

USER_AGENT = 'NewsBlur v1.0 - newsblur.com'

ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)

class Feed(models.Model):
    feed_address = models.URLField(max_length=255, verify_exists=True, unique=True)
    feed_link = models.URLField(max_length=1000, default="", blank=True, null=True)
    feed_title = models.CharField(max_length=255, default="", blank=True, null=True)
    feed_tagline = models.CharField(max_length=1024, default="", blank=True, null=True)
    active = models.BooleanField(default=True)
    num_subscribers = models.IntegerField(default=0)
    last_update = models.DateTimeField(auto_now=True)
    fetched_once = models.BooleanField(default=False)
    has_exception = models.BooleanField(default=False)
    min_to_decay = models.IntegerField(default=15)
    days_to_trim = models.IntegerField(default=90)
    creation = models.DateField(auto_now_add=True)
    etag = models.CharField(max_length=50, blank=True, null=True)
    last_modified = models.DateTimeField(null=True, blank=True)
    stories_last_month = models.IntegerField(default=0)
    average_stories_per_month = models.IntegerField(default=0)
    story_count_history = models.TextField(blank=True, null=True)
    next_scheduled_update = models.DateTimeField(default=datetime.datetime.now)
    last_load_time = models.IntegerField(default=0)
    popular_tags = models.CharField(max_length=1024, blank=True, null=True)
    popular_authors = models.CharField(max_length=2048, blank=True, null=True)
    
    
    def __unicode__(self):
        return self.feed_title

    def save(self, lock=None, *args, **kwargs):
        if self.feed_tagline and len(self.feed_tagline) > 1024:
            self.feed_tagline = self.feed_tagline[:1024]
            
        if lock:
            lock.acquire()
            try:
                super(Feed, self).save(*args, **kwargs)
            finally:
                lock.release()
        else:
            super(Feed, self).save(*args, **kwargs)
        
    def save_feed_history(self, status_code, message, exception=None):
        FeedFetchHistory.objects.create(feed=self, 
                                        status_code=status_code,
                                        message=message,
                                        exception=exception)
        old_fetch_histories = self.feed_fetch_history.all()[10:]
        for history in old_fetch_histories:
            history.delete()
            
        if status_code >= 400:
            fetch_history = self.feed_fetch_history.all().values('status_code')
            self.count_errors_in_history(fetch_history)
        
    def save_page_history(self, status_code, message, exception=None):
        PageFetchHistory.objects.create(feed=self, 
                                        status_code=status_code,
                                        message=message,
                                        exception=exception)
        old_fetch_histories = self.page_fetch_history.all()[10:]
        for history in old_fetch_histories:
            history.delete()
            
        if status_code >= 400:
            fetch_history = self.page_fetch_history.all().values('status_code')
            self.count_errors_in_history(fetch_history)
        
    def count_errors_in_history(self, fetch_history):
        non_errors = [h for h in fetch_history if int(h['status_code']) < 400]
        errors = [h for h in fetch_history if int(h['status_code']) >= 400]

        if len(non_errors) == 0 and len(errors) >= 1:
            self.has_exception = True
            self.active = False
            self.save()
    
    def count_subscribers(self, verbose=False, lock=None):
        from apps.reader.models import UserSubscription
        subs = UserSubscription.objects.filter(feed=self)
        self.num_subscribers = subs.count()

        self.save(lock=lock)
        
        if verbose:
            if self.num_subscribers <= 1:
                print '.',
            else:
                print "\n %s> %s subscriber%s: %s" % (
                    '-' * min(self.num_subscribers, 20),
                    self.num_subscribers,
                    '' if self.num_subscribers == 1 else 's',
                    self.feed_title,
                ),

    def count_stories(self, verbose=False, lock=None):
        month_ago = datetime.datetime.now() - datetime.timedelta(days=30)
        stories_last_month = Story.objects.filter(story_feed=self, story_date__gte=month_ago).count()
        self.stories_last_month = stories_last_month
        
{'fraction_non_alphanumeric': 0.08821876755946806, 'fraction_numerical': 0.012549166510582506, 'mean_word_length': 2.7118526242613834, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2097878', 'n_tokens_mistral': 3138, 'n_tokens_neox': 3016, 'n_words': 592}