<commit_before>from apps.rss_feeds.models import Feed, FeedUpdateHistory
# from apps.rss_feeds.models import FeedXML
from django.core.cache import cache
from django.conf import settings
from apps.reader.models import UserSubscription
from apps.rss_feeds.importer import PageImporter
from utils import feedparser
from django.db import IntegrityError
from utils.story_functions import pre_process_story
from utils import log as logging
import sys
import time
import datetime
import traceback
import multiprocessing
import urllib2
import xml.sax
import socket
import pymongo

# Refresh feed code adapted from Feedjack.
# http://feedjack.googlecode.com

VERSION = '0.9'
URL = 'http://www.newsblur.com/'
USER_AGENT = 'NewsBlur Fetcher %s - %s' % (VERSION, URL)
SLOWFEED_WARNING = 10
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
FEED_OK, FEED_SAME, FEED_ERRPARSE, FEED_ERRHTTP, FEED_ERREXC = range(5)

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))
    
import threading
class TimeoutError(Exception): pass
def timelimit(timeout):
    """borrowed from web.py"""
    def _1(function):
        def _2(*args, **kw):
            class Dispatch(threading.Thread):
                def __init__(self):
                    threading.Thread.__init__(self)
                    self.result = None
                    self.error = None
                    
                    self.setDaemon(True)
                    self.start()

                def run(self):
                    try:
                        self.result = function(*args, **kw)
                    except:
                        self.error = sys.exc_info()

            c = Dispatch()
            c.join(timeout)
            if c.isAlive():
                raise TimeoutError, 'took too long'
            if c.error:
                raise c.error[0], c.error[1]
            return c.result
        return _2
    return _1
    
class FetchFeed:
    def __init__(self, feed, options):
        self.feed = feed
        self.options = options
        self.fpf = None
    
    # @timelimit(20)
    def fetch(self):
        """ Downloads and parses a feed.
        """
        socket.setdefaulttimeout(30)
        identity = self.get_identity()
        log_msg = u'%2s ---> Fetching %s (%d)' % (identity,
                                                 self.feed.feed_title,
                                                 self.feed.id)
        logging.debug(log_msg)
                                                 
        # Check if feed still needs to be updated
        feed = Feed.objects.get(pk=self.feed.pk)
        if feed.next_scheduled_update > datetime.datetime.now() and not self.options.get('force'):
            log_msg = u'        ---> Already fetched %s (%d)' % (self.feed.feed_title,
                                                                 self.feed.id)
            logging.debug(log_msg)
            feed.save_feed_history(303, "Already fetched")
            return FEED_SAME, None
        else:
            feed.set_next_scheduled_update()
            
        etag=self.feed.etag<commit_msg>Removing weird threading-style time limiting of feed fetching.<commit_after>from apps.rss_feeds.models import Feed, FeedUpdateHistory
# from apps.rss_feeds.models import FeedXML
from django.core.cache import cache
from django.conf import settings
from apps.reader.models import UserSubscription
from apps.rss_feeds.importer import PageImporter
from utils import feedparser
from django.db import IntegrityError
from utils.story_functions import pre_process_story
from utils import log as logging
import sys
import time
import datetime
import traceback
import multiprocessing
import urllib2
import xml.sax
import socket
import pymongo

# Refresh feed code adapted from Feedjack.
# http://feedjack.googlecode.com

VERSION = '0.9'
URL = 'http://www.newsblur.com/'
USER_AGENT = 'NewsBlur Fetcher %s - %s' % (VERSION, URL)
SLOWFEED_WARNING = 10
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
FEED_OK, FEED_SAME, FEED_ERRPARSE, FEED_ERRHTTP, FEED_ERREXC = range(5)

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))
    
    
class FetchFeed:
    def __init__(self, feed, options):
        self.feed = feed
        self.options = options
        self.fpf = None
    
    def fetch(self):
        """ Downloads and parses a feed.
        """
        socket.setdefaulttimeout(30)
        identity = self.get_identity()
        log_msg = u'%2s ---> Fetching %s (%d)' % (identity,
                                                 self.feed.feed_title,
                                                 self.feed.id)
        logging.debug(log_msg)
                                                 
        # Check if feed still needs to be updated
        feed = Feed.objects.get(pk=self.feed.pk)
        if feed.next_scheduled_update > datetime.datetime.now() and not self.options.get('force'):
            log_msg = u'        ---> Already fetched %s (%d)' % (self.feed.feed_title,
                                                                 self.feed.id)
            logging.debug(log_msg)
            feed.save_feed_history(303, "Already fetched")
            return FEED_SAME, None
        else:
            feed.set_next_scheduled_update()
            
        etag=self.feed.etag
{'fraction_non_alphanumeric': 0.08752533628155519, 'fraction_numerical': 0.0062649714391007924, 'mean_word_length': 1.9073379753615425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6236000', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1519, 'n_words': 391}