<commit_before>import argparse
import json, urllib, httplib
import re
import MySQLdb
import datetime
import sys
import time

branch_paths = {
    'mozilla-central': 'mozilla-central',
    'mozilla-inbound': 'integration/mozilla-inbound'
}

branches = ['mozilla-central']

def getCSetResults(branch, revision):
    """
      https://tbpl.mozilla.org/php/getRevisionBuilds.php?branch=mozilla-inbound&rev=3435df09ce34

      no caching as data will change over time.  Some results will be in asap, others will take
      up to 12 hours (usually < 4 hours)
    """
    conn = httplib.HTTPSConnection('tbpl.mozilla.org')
    cset = "/php/getRevisionBuilds.php?branch=%s&rev=%s" % (branch, revision)
    conn.request("GET", cset)
    response = conn.getresponse()

    data = response.read()
    response.close()
    cdata = json.loads(data)
    return cdata

def getPushLog(branch, startdate):
    """
      https://hg.mozilla.org/integration/mozilla-inbound/pushlog?startdate=2013-06-19
    """

    conn = httplib.HTTPSConnection('hg.mozilla.org')
    pushlog = "/%s/pushlog?startdate=%04d-%02d-%02d" % (branch_paths[branch], startdate.year, startdate.month, startdate.day)
    conn.request("GET", pushlog)
    response = conn.getresponse()

    data = response.read()
    response.close()

    pushes = []
    csetid = re.compile('.*Changeset ([0-9a-f]{12}).*')
    dateid = re.compile('.*([0-9]{4}\-[0-9]{2}\-[0-9]{2})T([0-9\:]+)Z.*')
    push = None
    date = None
    for line in data.split('\n'):
        matches = csetid.match(line)
        if matches:
            push = matches.group(1)

        matches = dateid.match(line)
        if matches:
            date = "%s %s" % (matches.group(1), matches.group(2))

        if push and date:
            pushes.append([push, date])
            push = None
            date = None
    return pushes

def parseBuilder(buildername, branch):
    parts = buildername.split(branch)
    platform = parts[0]
    buildtype = branch.join(parts[1:])
    types = buildtype.split('test')
    buildtype, testtype = types[0].strip(), 'test'.join(types[1:]).strip()
    buildtype = buildtype.strip()
    if buildtype not in ['opt', 'debug', 'build'] and not testtype:
        testtype = buildtype
        buildtype = 'opt'

    platforms = ['linux32', 'linux32', 'linux32', 'linux32', 'linux64', 'linux64', 'linux64', 'linux64', 'osx10.6', 'osx10.7', 'osx10.7', 'osx10.8', 'winxp', 'winxp', 'win7', 'win764', 'win8', 'android-armv6', 'android-armv6', 'android-x86', 'android2.2', 'android-no-ion', 'android4.0', 'b2g-vm', 'b2g-emulator']

    platformtext = ['Linux', 'Ubuntu HW 12.04', 'Ubuntu VM 12.04', 'Rev3 Fedora 12', 'Linux x86-64', 'Rev3 Fedora 12x64', 'Ubuntu VM 12.04 x64', 'Ubuntu HW 12.04 x64', 'Rev4 MacOSX Snow Leopard 10.6', 'Rev4 MacOSX Lion 10.7', 'OS X 10.7', 'Rev5 MacOSX Mountain Lion 10.8', 'WINNT 5.2', 'Windows XP 32-bit', 'Windows 7 32-bit', 'Windows 7 64-bit', 'WINNT 6.2', 'Android Armv6', 'Android Armv6 Tegra 250', 'Android X86', 'Android Tegra 250', 'Android no-ionmonkey', 'Android 4.0 Panda', 'b2g_ics_armv7a_gecko_emulator_vm', 'b2g_ics_armv7a_gecko_emulator']

    index = 0
    for p in platformtext:
        if re.match(p, platform.strip()):
            break;
        index += 1
    if index < len(platforms):
        platform = platforms[index]
    else:
        return '', '', ''

    return platform, buildtype, testtype

def clearResults(branch, startdate):
    db = MySQLdb.connect(host="localhost",
                         user="root",
                         passwd="<PASSWORD>",
                         db="ouija")

    cur = db.cursor()
    cur.execute('delete from testjobs where branch="%s" and date >= "%04d-%02d-%02d"' % (branch, startdate.year, startdate.month, startdate.day)) 
    cur.close()

def uploadResults(data, branch, revision, date):
    db = MySQLdb.connect(host="localhost",
                         user="root",
                         passwd="<PASSWORD>",
                         db="ouija")

    cur = db.cursor()<commit_msg>Fix Android 2.2 platform text.<commit_after>import argparse
import json, urllib, httplib
import re
import MySQLdb
import datetime
import sys
import time

branch_paths = {
    'mozilla-central': 'mozilla-central',
    'mozilla-inbound': 'integration/mozilla-inbound'
}

branches = ['mozilla-central']

def getCSetResults(branch, revision):
    """
      https://tbpl.mozilla.org/php/getRevisionBuilds.php?branch=mozilla-inbound&rev=3435df09ce34

      no caching as data will change over time.  Some results will be in asap, others will take
      up to 12 hours (usually < 4 hours)
    """
    conn = httplib.HTTPSConnection('tbpl.mozilla.org')
    cset = "/php/getRevisionBuilds.php?branch=%s&rev=%s" % (branch, revision)
    conn.request("GET", cset)
    response = conn.getresponse()

    data = response.read()
    response.close()
    cdata = json.loads(data)
    return cdata

def getPushLog(branch, startdate):
    """
      https://hg.mozilla.org/integration/mozilla-inbound/pushlog?startdate=2013-06-19
    """

    conn = httplib.HTTPSConnection('hg.mozilla.org')
    pushlog = "/%s/pushlog?startdate=%04d-%02d-%02d" % (branch_paths[branch], startdate.year, startdate.month, startdate.day)
    conn.request("GET", pushlog)
    response = conn.getresponse()

    data = response.read()
    response.close()

    pushes = []
    csetid = re.compile('.*Changeset ([0-9a-f]{12}).*')
    dateid = re.compile('.*([0-9]{4}\-[0-9]{2}\-[0-9]{2})T([0-9\:]+)Z.*')
    push = None
    date = None
    for line in data.split('\n'):
        matches = csetid.match(line)
        if matches:
            push = matches.group(1)

        matches = dateid.match(line)
        if matches:
            date = "%s %s" % (matches.group(1), matches.group(2))

        if push and date:
            pushes.append([push, date])
            push = None
            date = None
    return pushes

def parseBuilder(buildername, branch):
    parts = buildername.split(branch)
    platform = parts[0]
    buildtype = branch.join(parts[1:])
    types = buildtype.split('test')
    buildtype, testtype = types[0].strip(), 'test'.join(types[1:]).strip()
    buildtype = buildtype.strip()
    if buildtype not in ['opt', 'debug', 'build'] and not testtype:
        testtype = buildtype
        buildtype = 'opt'

    platforms = ['linux32', 'linux32', 'linux32', 'linux32', 'linux64', 'linux64', 'linux64', 'linux64', 'osx10.6', 'osx10.7', 'osx10.7', 'osx10.8', 'winxp', 'winxp', 'win7', 'win764', 'win8', 'android-armv6', 'android-armv6', 'android-x86', 'android2.2', 'android-no-ion', 'android4.0', 'b2g-vm', 'b2g-emulator']

    platformtext = ['Linux', 'Ubuntu HW 12.04', 'Ubuntu VM 12.04', 'Rev3 Fedora 12', 'Linux x86-64', 'Rev3 Fedora 12x64', 'Ubuntu VM 12.04 x64', 'Ubuntu HW 12.04 x64', 'Rev4 MacOSX Snow Leopard 10.6', 'Rev4 MacOSX Lion 10.7', 'OS X 10.7', 'Rev5 MacOSX Mountain Lion 10.8', 'WINNT 5.2', 'Windows XP 32-bit', 'Windows 7 32-bit', 'Windows 7 64-bit', 'WINNT 6.2', 'Android Armv6', 'Android Armv6 Tegra 250', 'Android X86', 'Android 2.2 Tegra', 'Android no-ionmonkey', 'Android 4.0 Panda', 'b2g_ics_armv7a_gecko_emulator_vm', 'b2g_ics_armv7a_gecko_emulator']

    index = 0
    for p in platformtext:
        if re.match(p, platform.strip()):
            break;
        index += 1
    if index < len(platforms):
        platform = platforms[index]
    else:
        return '', '', ''

    return platform, buildtype, testtype

def clearResults(branch, startdate):
    db = MySQLdb.connect(host="localhost",
                         user="root",
                         passwd="<PASSWORD>",
                         db="ouija")

    cur = db.cursor()
    cur.execute('delete from testjobs where branch="%s" and date >= "%04d-%02d-%02d"' % (branch, startdate.year, startdate.month, startdate.day)) 
    cur.close()

def uploadResults(data, branch, revision, date):
    db = MySQLdb.connect(host="localhost",
                         user="root",
                         passwd="<PASSWORD>",
                         db="ouija")

    cur = db.cursor()
{'fraction_non_alphanumeric': 0.1607474322484841, 'fraction_numerical': 0.04294023016953347, 'mean_word_length': 2.878119001919386, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6462261', 'n_tokens_mistral': 2986, 'n_tokens_neox': 2690, 'n_words': 633}