<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
politwoops-worker.py

Created by <NAME> on 2010-05-30.
Copyright (c) 2010 __MyCompanyName__. All rights reserved.
"""

import sys
import getopt
import ConfigParser
import MySQLdb

from time import sleep

import anyjson
import beanstalkc

import socket
# disable buffering
socket._fileobject.default_bufsize = 0

import httplib
httplib.HTTPConnection.debuglevel = 1

import urllib
import urllib2

# external libs
sys.path.insert(0, './lib')

import tweetsclient

import politwoops

from stathat import StatHat

help_message = '''
The help message goes here.
'''

# used for screenshot capturing and archiving
import subprocess
import hashlib
from boto.s3.connection import S3Connection
from boto.s3.key import Key

class Usage(Exception):
    def __init__(self, msg):
        self.msg = msg


class DeletedTweetsWorker:
    def __init__(self, verbose, output, images):
        self.verbose = verbose
        self.output = output
        self.images = images
        self.get_config()
    
    def _debug(self, msg):
        if self.verbose:
            print >>sys.stderr, msg

    def get_database(self):
        self._debug("Making DB connection")
        conn = MySQLdb.connect(
            host=self.config.get('database', 'host'),
            port=int(self.config.get('database', 'port')),
            db=self.config.get('database', 'database'),
            user=self.config.get('database', 'username'),
            passwd=self.config.get('database', 'password'),
            charset="utf8",
            use_unicode=True
        )
        conn.autocommit(True) # needed if you're using InnoDB
        conn.cursor().execute('SET NAMES UTF8')
        return conn
    
    def get_beanstalk(self):
        tube = self.config.get('beanstalk', 'tube')
        self._debug("Initiating beanstalk connection for tube %s ..." % (tube))
        beanstalk = politwoops.utils.beanstalk(host=self.config.get('beanstalk', 'host'), port=int(self.config.get('beanstalk', 'port')), tube=self.config.get('beanstalk', 'tube'))
        return beanstalk

    def get_config(self):
        self._debug("Reading config ...")
        self.config = tweetsclient.Config().get()

    def get_users(self):
        cursor = self.database.cursor()
        q = "SELECT `twitter_id`, `user_name`, `id` FROM `politicians`"
        cursor.execute(q)
        ids = {}
        politicians = {}
        for t in cursor.fetchall():
            ids[t[0]] = t[2]
            politicians[t[0]] = t[1]
        self._debug("Found ids : ")
        self._debug(ids)
        self._debug("Found politicians : ")
        self._debug(politicians)
        return ids, politicians

    def run(self):
        self.database = self.get_database()
        self.beanstalk = self.get_beanstalk()
        self.users, self.politicians = self.get_users()
        self.stathat = self.get_stathat()
        while True:
            sleep(0.2)
            job = self.beanstalk.reserve(timeout=0)
            if job:
                self.handle_tweet(job.body)
                job.delete()

    def run_with_restart(self):
        # keeps tabs on whether we should restart the connection to Twitter ourselves
        shouldRestart = True
        # keeps tabs on how many times we've unsuccesfully restarted -- more means longer waiting times
        self.restartCounter = 0
        
        while shouldRestart:
            shouldRestart = False
            try:
                self.run()
            except Exception as e:
                shouldRestart = True
                sleep(1) # restart after a second

                self.restartCounter += 1
                self._debug("Some sort of error, restarting for the %s time...\n\t%s" % (self.restartCounter, e.message))
    
    def get_stathat(self):
        stathat_enabled = (self.config.get('stathat', 'enabled') == 'yes')
        if not stathat_enabled:
            self._debug('Running without stathat ...')
            return
        else:
            self._debug('StatHat ingeration enabled ...')
            return StatHat()
        
    def stathat_add_count(self, stat_name):
        if self.stathat is not None:
            try:
                self.stathat.ez_post_count(self.config.get('stathat', 'email'), stat_name, 1)
            except urllib2.URLError, e:
                pass
            except urllib2.HTTPError, e:
                pass
    
    def handle_tweet(self, job_body):
        tweet = anyjson.deserialize(job_body)
        if tweet.has_key('delete'):
            if tweet['delete']['status']['user_id'] in self.users.keys():
                self.handle_deletion(tweet)
        else:
            if tweet.has_key('user') and (tweet['user']['id'] in self.users.keys()):
                self.handle_new(tweet)

                if self.images and tweet.has_key('entities'):
                    tweet_id = tweet['id']

                    for i, url_entity in enumerate(tweet['entities']['urls']):
                        origin_url = url_entity['expanded_url']
                        filename = "%i-%i.png" % (tweet_id, i)

                        if not origin_url:
                            self._debug("Didn't get an expanded URL, skipping this image")
                            return
<commit_msg>If the expanded URL is missing (?) use the t.co version, now that rasterize.js follows redirects<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
politwoops-worker.py

Created by <NAME> on 2010-05-30.
Copyright (c) 2010 __MyCompanyName__. All rights reserved.
"""

import sys
import getopt
import ConfigParser
import MySQLdb

from time import sleep

import anyjson
import beanstalkc

import socket
# disable buffering
socket._fileobject.default_bufsize = 0

import httplib
httplib.HTTPConnection.debuglevel = 1

import urllib
import urllib2

# external libs
sys.path.insert(0, './lib')

import tweetsclient

import politwoops

from stathat import StatHat

help_message = '''
The help message goes here.
'''

# used for screenshot capturing and archiving
import subprocess
import hashlib
from boto.s3.connection import S3Connection
from boto.s3.key import Key

class Usage(Exception):
    def __init__(self, msg):
        self.msg = msg


class DeletedTweetsWorker:
    def __init__(self, verbose, output, images):
        self.verbose = verbose
        self.output = output
        self.images = images
        self.get_config()
    
    def _debug(self, msg):
        if self.verbose:
            print >>sys.stderr, msg

    def get_database(self):
        self._debug("Making DB connection")
        conn = MySQLdb.connect(
            host=self.config.get('database', 'host'),
            port=int(self.config.get('database', 'port')),
            db=self.config.get('database', 'database'),
            user=self.config.get('database', 'username'),
            passwd=self.config.get('database', 'password'),
            charset="utf8",
            use_unicode=True
        )
        conn.autocommit(True) # needed if you're using InnoDB
        conn.cursor().execute('SET NAMES UTF8')
        return conn
    
    def get_beanstalk(self):
        tube = self.config.get('beanstalk', 'tube')
        self._debug("Initiating beanstalk connection for tube %s ..." % (tube))
        beanstalk = politwoops.utils.beanstalk(host=self.config.get('beanstalk', 'host'), port=int(self.config.get('beanstalk', 'port')), tube=self.config.get('beanstalk', 'tube'))
        return beanstalk

    def get_config(self):
        self._debug("Reading config ...")
        self.config = tweetsclient.Config().get()

    def get_users(self):
        cursor = self.database.cursor()
        q = "SELECT `twitter_id`, `user_name`, `id` FROM `politicians`"
        cursor.execute(q)
        ids = {}
        politicians = {}
        for t in cursor.fetchall():
            ids[t[0]] = t[2]
            politicians[t[0]] = t[1]
        self._debug("Found ids : ")
        self._debug(ids)
        self._debug("Found politicians : ")
        self._debug(politicians)
        return ids, politicians

    def run(self):
        self.database = self.get_database()
        self.beanstalk = self.get_beanstalk()
        self.users, self.politicians = self.get_users()
        self.stathat = self.get_stathat()
        while True:
            sleep(0.2)
            job = self.beanstalk.reserve(timeout=0)
            if job:
                self.handle_tweet(job.body)
                job.delete()

    def run_with_restart(self):
        # keeps tabs on whether we should restart the connection to Twitter ourselves
        shouldRestart = True
        # keeps tabs on how many times we've unsuccesfully restarted -- more means longer waiting times
        self.restartCounter = 0
        
        while shouldRestart:
            shouldRestart = False
            try:
                self.run()
            except Exception as e:
                shouldRestart = True
                sleep(1) # restart after a second

                self.restartCounter += 1
                self._debug("Some sort of error, restarting for the %s time...\n\t%s" % (self.restartCounter, e.message))
    
    def get_stathat(self):
        stathat_enabled = (self.config.get('stathat', 'enabled') == 'yes')
        if not stathat_enabled:
            self._debug('Running without stathat ...')
            return
        else:
            self._debug('StatHat ingeration enabled ...')
            return StatHat()
        
    def stathat_add_count(self, stat_name):
        if self.stathat is not None:
            try:
                self.stathat.ez_post_count(self.config.get('stathat', 'email'), stat_name, 1)
            except urllib2.URLError, e:
                pass
            except urllib2.HTTPError, e:
                pass
    
    def handle_tweet(self, job_body):
        tweet = anyjson.deserialize(job_body)
        if tweet.has_key('delete'):
            if tweet['delete']['status']['user_id'] in self.users.keys():
                self.handle_deletion(tweet)
        else:
            if tweet.has_key('user') and (tweet['user']['id'] in self.users.keys()):
                self.handle_new(tweet)

                if self.images and tweet.has_key('entities'):
                    tweet_id = tweet['id']

                    for i, url_entity in enumerate(tweet['entities']['urls']):
                        origin_url = url_entity['expanded_url']
                        filename = "%i-%i.png" % (tweet_id, i)

                        if not origin_url:
                            self._debug("Didn't get an expanded URL, using the t.co URL")
                            origin_url = url_entity['url']

                        if not origin_url:
                            self._debug("Apparently no t.co URL either, whatever, skipping")
                            return

{'fraction_non_alphanumeric': 0.11063006632277082, 'fraction_numerical': 0.006448047162859248, 'mean_word_length': 2.0531496062992125, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '84073', 'n_tokens_mistral': 3253, 'n_tokens_neox': 3073, 'n_words': 769}