<commit_before>#!/usr/bin/env python

UA = "diffengine/0.1 (+https://github.com/edsu/diffengine)"

import os
import json
import time
import yaml
import bleach
import codecs
import jinja2
import tweepy
import logging
import htmldiff
import requests
import selenium
import feedparser
import readability

from peewee import *
from datetime import datetime, timedelta
from selenium import webdriver

config = yaml.load(open('config.yaml'))
db = SqliteDatabase('diffengine.db')

twitter = None
if 'twitter' in config:
    t = config['twitter']
    auth = tweepy.OAuthHandler(t['consumer_key'], t['consumer_secret'])
    auth.secure = True
    auth.set_access_token(t['access_token'], t['access_token_secret'])
    twitter = tweepy.API(auth)
            

class Feed(Model):
    url = CharField(primary_key=True)
    name = CharField()
    created = DateTimeField(default=datetime.utcnow)

    def get_latest(self):
        log = logging.getLogger(__name__)
        log.debug("fetching feed: %s", self.url)
        feed = feedparser.parse(self.url)
        for e in feed.entries:
            entry, created = Entry.get_or_create(url=e.link, feed=self)
            if created:
                log.debug("found new entry: %s", e.link)

    class Meta:
        database = db


class Entry(Model):
    url = CharField()
    created = DateTimeField(default=datetime.utcnow)
    checked = DateTimeField(default=datetime.utcnow)
    feed = ForeignKeyField(Feed, related_name='entries')

    def stale(self):
        """
        A heuristic for checking new content very often, and checking 
        older content less frequently. If an entry is deemed stale then
        it is worth checking again to see if the content has changed.
        """
        log = logging.getLogger(__name__)

        # never been checked before it's obviously stale
        if not self.checked:
            return True

        # time since the entry was created
        hotness = (datetime.utcnow() - self.created).seconds
        if hotness == 0:
            return True

        # time since the entry was last checked
        staleness = (datetime.utcnow() - self.checked).seconds

        # ratio of staleness to hotness
        r = staleness / float(hotness)

        # TODO: allow this magic number to be configured per feed?
        if r >= 0.2:
            log.debug("%s is stale (r=%f)", self.url, r)
            return True

        log.debug("%s not stale (r=%f)", self.url, r)
        return False

    def get_latest(self):
        log = logging.getLogger(__name__)

        if not self.stale():
            return

        # make sure we don't go too fast
        time.sleep(1)

        # fetch the current readability-ized content for the page
        log.debug("checking %s", self.url)
        resp = requests.get(self.url, headers={"User-Agent": UA})
        doc = readability.Document(resp.text)
        title = doc.title()
        summary = doc.summary(html_partial=True)
        summary = bleach.clean(summary, tags=["p"], strip=True)


        # get the latest version, if we have one
        versions = EntryVersion.select().where(EntryVersion.entry==self)
        versions = versions.order_by(-EntryVersion.created)
        if len(versions) == 0:
            old = None
        else:
            old = versions[0]

        # compare what we got against the latest version and create a 
        # new version if it looks different, or is brand new (no old version)
        diff = None

        if not old or old.title != title or old.summary != summary:
            new = EntryVersion.create(
                title=title,
                summary=summary,
                entry=self
            )
            new.archive()
            if old:
                log.info("found new version %s", old.entry.url)
                diff = Diff.create(old=old, new=new)
                diff.generate()
            else:
                log.debug("found first version: %s", self.url)

        self.checked = datetime.utcnow()
        self.save()
        return diff

    class Meta:
        database = db


class EntryVersion(Model):
    title = CharField()
    summary = CharField()
    created = DateTimeField(default=datetime.utcnow)
    archive_url = CharField(null=True)
    entry = ForeignKeyField(Entry, related_name='versions')

    @property
    def html(self):
        return "<h1>%s</h1>\n\n%s" % (self.title, self.summary)

    def archive(self):
        log = logging.getLogger(__name__)
        resp = requests.post('https://pragma.archivelab.org',
                             json={'url': self.entry.url},
                             headers={"User-Agent": UA})
        data = resp.json()
        if 'wayback_id' not in data:
            log.error("unexpected archive.org response: %s", json.dumps(data))
            return
        wayback_id = data['wayback_id']
        self.archive_url = "https://wayback.archive.org" + wayback_id
        log.debug("archived version at %s", self.archive_url)
        self.save()

    class Meta:
        database = db


class Diff(Model):
    old = ForeignKeyField(EntryVersion, related_name="prev_diff")
    new = ForeignKeyField(EntryVersion, related_name="next_diff")
    created = DateTimeField(default=datetime.utcnow)
    tweeted = DateTimeField(null=True)
    blogged = DateTimeField(null=True)

    @property
    def html_path(self):
        dir_name = "diffs/%s" % (self.old.entry.id % 53)
        if not os.path.isdir(dir_name):
            os.makedirs(dir_name)
        return "%s/%s-%s.html" % (dir_name, self.old.id, self.new.id)

    @property
    def screenshot_path(self):
        return self.html_path.replace(".html", ".jpg")

    def generate(self):
        self._generate_diff_html()
        self._generate_diff_image()

    def _generate_diff_html(self):
        log = logging.getLogger(__name__)
        if os.path.isfile(self.html_path):
            return
        log.debug("creating html diff: %s", self.html_path)
        diff = htmldiff.render_html_diff(self.old.html, self.new.html)<commit_msg>larger prime<commit_after>#!/usr/bin/env python

UA = "diffengine/0.1 (+https://github.com/edsu/diffengine)"

import os
import json
import time
import yaml
import bleach
import codecs
import jinja2
import tweepy
import logging
import htmldiff
import requests
import selenium
import feedparser
import readability

from peewee import *
from datetime import datetime, timedelta
from selenium import webdriver

config = yaml.load(open('config.yaml'))
db = SqliteDatabase('diffengine.db')

twitter = None
if 'twitter' in config:
    t = config['twitter']
    auth = tweepy.OAuthHandler(t['consumer_key'], t['consumer_secret'])
    auth.secure = True
    auth.set_access_token(t['access_token'], t['access_token_secret'])
    twitter = tweepy.API(auth)
            

class Feed(Model):
    url = CharField(primary_key=True)
    name = CharField()
    created = DateTimeField(default=datetime.utcnow)

    def get_latest(self):
        log = logging.getLogger(__name__)
        log.debug("fetching feed: %s", self.url)
        feed = feedparser.parse(self.url)
        for e in feed.entries:
            entry, created = Entry.get_or_create(url=e.link, feed=self)
            if created:
                log.debug("found new entry: %s", e.link)

    class Meta:
        database = db


class Entry(Model):
    url = CharField()
    created = DateTimeField(default=datetime.utcnow)
    checked = DateTimeField(default=datetime.utcnow)
    feed = ForeignKeyField(Feed, related_name='entries')

    def stale(self):
        """
        A heuristic for checking new content very often, and checking 
        older content less frequently. If an entry is deemed stale then
        it is worth checking again to see if the content has changed.
        """
        log = logging.getLogger(__name__)

        # never been checked before it's obviously stale
        if not self.checked:
            return True

        # time since the entry was created
        hotness = (datetime.utcnow() - self.created).seconds
        if hotness == 0:
            return True

        # time since the entry was last checked
        staleness = (datetime.utcnow() - self.checked).seconds

        # ratio of staleness to hotness
        r = staleness / float(hotness)

        # TODO: allow this magic number to be configured per feed?
        if r >= 0.2:
            log.debug("%s is stale (r=%f)", self.url, r)
            return True

        log.debug("%s not stale (r=%f)", self.url, r)
        return False

    def get_latest(self):
        log = logging.getLogger(__name__)

        if not self.stale():
            return

        # make sure we don't go too fast
        time.sleep(1)

        # fetch the current readability-ized content for the page
        log.debug("checking %s", self.url)
        resp = requests.get(self.url, headers={"User-Agent": UA})
        doc = readability.Document(resp.text)
        title = doc.title()
        summary = doc.summary(html_partial=True)
        summary = bleach.clean(summary, tags=["p"], strip=True)


        # get the latest version, if we have one
        versions = EntryVersion.select().where(EntryVersion.entry==self)
        versions = versions.order_by(-EntryVersion.created)
        if len(versions) == 0:
            old = None
        else:
            old = versions[0]

        # compare what we got against the latest version and create a 
        # new version if it looks different, or is brand new (no old version)
        diff = None

        if not old or old.title != title or old.summary != summary:
            new = EntryVersion.create(
                title=title,
                summary=summary,
                entry=self
            )
            new.archive()
            if old:
                log.info("found new version %s", old.entry.url)
                diff = Diff.create(old=old, new=new)
                diff.generate()
            else:
                log.debug("found first version: %s", self.url)

        self.checked = datetime.utcnow()
        self.save()
        return diff

    class Meta:
        database = db


class EntryVersion(Model):
    title = CharField()
    summary = CharField()
    created = DateTimeField(default=datetime.utcnow)
    archive_url = CharField(null=True)
    entry = ForeignKeyField(Entry, related_name='versions')

    @property
    def html(self):
        return "<h1>%s</h1>\n\n%s" % (self.title, self.summary)

    def archive(self):
        log = logging.getLogger(__name__)
        resp = requests.post('https://pragma.archivelab.org',
                             json={'url': self.entry.url},
                             headers={"User-Agent": UA})
        data = resp.json()
        if 'wayback_id' not in data:
            log.error("unexpected archive.org response: %s", json.dumps(data))
            return
        wayback_id = data['wayback_id']
        self.archive_url = "https://wayback.archive.org" + wayback_id
        log.debug("archived version at %s", self.archive_url)
        self.save()

    class Meta:
        database = db


class Diff(Model):
    old = ForeignKeyField(EntryVersion, related_name="prev_diff")
    new = ForeignKeyField(EntryVersion, related_name="next_diff")
    created = DateTimeField(default=datetime.utcnow)
    tweeted = DateTimeField(null=True)
    blogged = DateTimeField(null=True)

    @property
    def html_path(self):
        # use prime number to spread across directories
        dir_name = "diffs/%s" % (self.old.entry.id % 257)
        if not os.path.isdir(dir_name):
            os.makedirs(dir_name)
        return "%s/%s-%s.html" % (dir_name, self.old.id, self.new.id)

    @property
    def screenshot_path(self):
        return self.html_path.replace(".html", ".jpg")

    def generate(self):
        self._generate_diff_html()
        self._generate_diff_image()

    def _generate_diff_html(self):
        log = logging.getLogger(__name__)
        if os.path.isfile(self.html_path):
            return
        log.debug("creating html diff: %s", self.html_path)
        diff = htmldiff.render_html_diff(self.old.html, self.new.html)
{'fraction_non_alphanumeric': 0.10763032395987503, 'fraction_numerical': 0.002220029600394672, 'mean_word_length': 2.2962059620596205, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '598910', 'n_tokens_mistral': 3609, 'n_tokens_neox': 3471, 'n_words': 937}