<commit_before># coding=utf-8

from webapp2 import RequestHandler
import urllib
from google.appengine.api.urlfetch import fetch
from google.appengine.api import taskqueue
import hmac
from hashlib import sha1
from base64 import b64encode
from time import time
from random import getrandbits
import json
import logging

api_urls = {
    'request_token_url': 'http://twitter.com/oauth/request_token',
    'access_token_url' : 'http://twitter.com/oauth/access_token',
    'authorize_url'    : 'http://twitter.com/oauth/authorize',
    'tweet_url'        : 'https://api.twitter.com/1.1/statuses/update.json',
    'github_commits'   : 'https://api.github.com/repos/defeo/defeo.github.io/commits/',
    'blog_base'        : 'http://defeo.lu',
    'blog_posts'       : '/api/posts.json'
    }

# Configuration variables
entry_url   = '/tweet-ghblog'
worker_url  = '/tweet-ghblog/commit'
def w_config(m):
    def wrap(self):
        self._post_prefix = self.app.config['tweet_blog'].get('post_prefix') or '[POST]'
        self._post_dir    = self.app.config['tweet_blog'].get('post_dir')    or '_posts/'
        m(self)
    return wrap
        
class CheckCommits(RequestHandler):
    '''
    This hook repsonds to post requests from the github Webhook
    service. It looks for new post commits inside the commits posted
    by github.
    '''
    @w_config
    def post(self):
        commits = json.loads(self.request.get('payload'))
        # Examine each commit in the payload of the request
        count = 0
        for c in commits["commits"]:
            if c['message'].startswith(self._post_prefix):
                # If commit starts with the right prefix, launch a task to examine it
                taskqueue.add(url=worker_url, params={'commit-id': c['id']})
                count += 1
                
        logging.info('Considering %d commits.' % count)
        self.response.content_type = 'application/json'
        self.response.write('{"commits": %d}' % count)


class Tweet(RequestHandler):
    '''
    This hook examines a single commit, looks for newly added files,
    and tweets any new post found.
    '''
    @w_config
    def post(self):
        # Fetch and parse json summary of posts from the blog
        posts = fetch(api_urls['blog_base'] + api_urls['blog_posts'])
        if posts.status_code != 200:
            logging.error('Cannot fetch post list. %s' % post.content)
            self.abort(500, detail='Cannot fetch post list.')
        posts = json.loads(posts.content)

        # Fetch and parse commit data from github
        commit = fetch(api_urls['github_commits'] + self.request.get('commit-id'),
                       validate_certificate=True)
        if commit.status_code != 200:
            logging.error('Cannot fetch commit data. %s' % commit.content)
            self.abort(500, 'Cannot fetch commit data.')
        commit = json.loads(commit.content)

        if commit['commit']['message'].startswith(self._post_prefix):
            # If commit starts with right prefix, look for new files in the post directory
            count = 0
            for f in commit['files']:
                if f['status'] == 'added' and f['filename'].startswith(self._post_dir):
                    count += 1
                    # Tweet post announcement
                    post = posts.get(f['filename'])
                    if post is None:
                        logging.error('Post %s missing in list.' % f['filename'])
                        self.abort(500, 'Post seems to be missing in list.')

                    tweet = u'#Blog post %s.%s %s%s' % (post['title'],
                                                        ''.join(' #' + t for t in post['tags'][:2]),
                                                         api_urls['blog_base'], post['url'])<commit_msg>GitHub API call now use authentication to avoid rate limiting<commit_after># coding=utf-8

from webapp2 import RequestHandler
import urllib
from google.appengine.api.urlfetch import fetch
from google.appengine.api import taskqueue
import hmac
from hashlib import sha1
from base64 import b64encode
from time import time
from random import getrandbits
import json
import logging

api_urls = {
    'request_token_url': 'http://twitter.com/oauth/request_token',
    'access_token_url' : 'http://twitter.com/oauth/access_token',
    'authorize_url'    : 'http://twitter.com/oauth/authorize',
    'tweet_url'        : 'https://api.twitter.com/1.1/statuses/update.json',
    'github_commits'   : 'https://api.github.com/repos/defeo/defeo.github.io/commits/',
    'blog_base'        : 'http://defeo.lu',
    'blog_posts'       : '/api/posts.json'
    }

# Configuration variables
entry_url   = '/tweet-ghblog'
worker_url  = '/tweet-ghblog/commit'
def w_config(m):
    def wrap(self):
        self._post_prefix = self.app.config['tweet_blog'].get('post_prefix') or '[POST]'
        self._post_dir    = self.app.config['tweet_blog'].get('post_dir')    or '_posts/'
        m(self)
    return wrap
        
class CheckCommits(RequestHandler):
    '''
    This hook repsonds to post requests from the github Webhook
    service. It looks for new post commits inside the commits posted
    by github.
    '''
    @w_config
    def post(self):
        commits = json.loads(self.request.get('payload'))
        # Examine each commit in the payload of the request
        count = 0
        for c in commits["commits"]:
            if c['message'].startswith(self._post_prefix):
                # If commit starts with the right prefix, launch a task to examine it
                taskqueue.add(url=worker_url, params={'commit-id': c['id']})
                count += 1
                
        logging.info('Considering %d commits.' % count)
        self.response.content_type = 'application/json'
        self.response.write('{"commits": %d}' % count)


class Tweet(RequestHandler):
    '''
    This hook examines a single commit, looks for newly added files,
    and tweets any new post found.
    '''
    @w_config
    def post(self):
        # Fetch and parse json summary of posts from the blog
        posts = fetch(api_urls['blog_base'] + api_urls['blog_posts'])
        if posts.status_code != 200:
            logging.error('Cannot fetch post list. %s' % post.content)
            self.abort(500, detail='Cannot fetch post list.')
        posts = json.loads(posts.content)

        # Fetch and parse commit data from github
        commit = fetch((api_urls['github_commits'] + self.request.get('commit-id') + 
                        "?client_id=%s&client_secret=%s" % 
                        (self.app.config['tweet_blog']['github_id'],
                         self.app.config['tweet_blog']['github_secret'])),
                       validate_certificate=True)
        if commit.status_code != 200:
            logging.error('Cannot fetch commit data. %s' % commit.content)
            self.abort(500, 'Cannot fetch commit data.')
        commit = json.loads(commit.content)

        if commit['commit']['message'].startswith(self._post_prefix):
            # If commit starts with right prefix, look for new files in the post directory
            count = 0
            for f in commit['files']:
                if f['status'] == 'added' and f['filename'].startswith(self._post_dir):
                    count += 1
                    # Tweet post announcement
                    post = posts.get(f['filename'])
                    if post is None:
                        logging.error('Post %s missing in list.' % f['filename'])
                        self.abort(500, 'Post seems to be missing in list.')

                    tweet = u'#Blog post %s.%s %s%s' % (post['title'],
                                                        ''.join(' #' + t for t in post['tags'][:2]),
                                                         api_urls['blog_base'], post['url'])
{'fraction_non_alphanumeric': 0.11934051997463538, 'fraction_numerical': 0.007355738744451491, 'mean_word_length': 2.151878497202238, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4282259', 'n_tokens_mistral': 2293, 'n_tokens_neox': 2207, 'n_words': 597}