<commit_before>#! /usr/bin/python

'''
Saves relevant data fed back from TwitterStream etc next to its PID and timestamp ready for analysis
Needs to do limited analysis to work out which keywords in the tweet stream correspond to which programme
'''

from datetime import datetime
import os
import string
import time as time2
from time import time

from Axon.Ipc import producerFinished
from Axon.Ipc import shutdownMicroprocess
from Axon.ThreadedComponent import threadedcomponent
import MySQLdb
import _mysql_exceptions
import cjson
from dateutil.parser import parse
from Kamaelia.Apps.SocialBookmarks.Print import Print

class DataCollector(threadedcomponent):
    Inboxes = {
        "inbox" : "Receives data in the format [tweetjson,[pid,pid]]",
        "control" : ""
    }
    Outboxes = {
        "outbox" : "",
        "signal" : ""
    }

    def __init__(self,dbuser,dbpass):
        super(DataCollector, self).__init__()
        self.dbuser = dbuser
        self.dbpass = dbpass
        self.cursor = None  # xyz # dupe
        self.cursor_dupe = None  # xyz #dupe

    def finished(self):
        while self.dataReady("control"):
            msg = self.recv("control")
            if isinstance(msg, producerFinished) or isinstance(msg, shutdownMicroprocess):
                self.send(msg, "signal")
                return True
        return False

    def dbConnect(self):
        db = MySQLdb.connect(user=self.dbuser,passwd=self.dbpass,db="twitter_bookmarks",use_unicode=True,charset="utf8")
        cursor = db.cursor()  # xyz
        self.cursor = cursor  # xyz
        if 0:
            db_dupe = MySQLdb.connect(user=self.dbuser,passwd=self.dbpass,db="twitter_bookmarks_next",use_unicode=True,charset="utf8")
            cursor_dupe = db_dupe.cursor()   # xyz
            self.cursor_dupe = cursor_dupe   # xyz


    # The purpose of pulling these three out is to make it simpler to keep things in sync between multiple DBs
    def db_select(self,command, args=None):
        if args:
            self.cursor.execute(command,args) #xyz
        else:
            self.cursor.execute(command) #xyz
<commit_msg>Initial check that basic stuff works as wanted<commit_after>#! /usr/bin/python

'''
Saves relevant data fed back from TwitterStream etc next to its PID and timestamp ready for analysis
Needs to do limited analysis to work out which keywords in the tweet stream correspond to which programme
'''

from datetime import datetime
import os
import string
import time as time2
from time import time

from Axon.Ipc import producerFinished
from Axon.Ipc import shutdownMicroprocess
from Axon.ThreadedComponent import threadedcomponent
import MySQLdb
import _mysql_exceptions
import cjson
from dateutil.parser import parse
from Kamaelia.Apps.SocialBookmarks.Print import Print
from Kamaelia.Apps.SocialBookmarks.DBWrapper import DBWrapper

class DataCollector(DBWrapper,threadedcomponent):
    Inboxes = {
        "inbox" : "Receives data in the format [tweetjson,[pid,pid]]",
        "control" : ""
    }
    Outboxes = {
        "outbox" : "",
        "signal" : ""
    }

    def __init__(self,dbuser,dbpass):
        super(DataCollector, self).__init__(dbuser=dbuser,dbpass=dbpass)
        # These should all be set now by the DBWrapper
        # Ensure we crash if they're not...
        self.dbuser, self.dbpass, self.cursor, self.cursor_dupe
        if 0:
            self.dbuser = dbuser
            self.dbpass = dbpass
            self.cursor = None  # xyz # dupe
            self.cursor_dupe = None  # xyz #dupe

    def finished(self):
        while self.dataReady("control"):
            msg = self.recv("control")
            if isinstance(msg, producerFinished) or isinstance(msg, shutdownMicroprocess):
                self.send(msg, "signal")
                return True
        return False

    def dbConnect(self):
        db = MySQLdb.connect(user=self.dbuser,passwd=self.dbpass,db="twitter_bookmarks",use_unicode=True,charset="utf8")
        cursor = db.cursor()  # xyz
        self.cursor = cursor  # xyz
        if 0:
            db_dupe = MySQLdb.connect(user=self.dbuser,passwd=<PASSWORD>,db="twitter_bookmarks_next",use_unicode=True,charset="utf8")
            cursor_dupe = db_dupe.cursor()   # xyz
            self.cursor_dupe = cursor_dupe   # xyz


    # The purpose of pulling these three out is to make it simpler to keep things in sync between multiple DBs
    def db_select(self,command, args=None):
        if args:
            self.cursor.execute(command,args) #xyz
        else:
            self.cursor.execute(command) #xyz

{'fraction_non_alphanumeric': 0.08638743455497382, 'fraction_numerical': 0.001963350785340314, 'mean_word_length': 2.8594276094276094, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5451944', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1316, 'n_words': 386}