<commit_before>from __future__ import print_function
import re
import sys
import sqlite3
from pprint import pprint

import mysqlite as sql


sqlite_file = 'speedtest.sqlite'


def warning(*objs):
    print("WARNING: ", *objs, file=sys.stderr)


def convert_datetime(line):
    garbage, datetime = line.split(' ')
    return datetime


def convert_speed(line):
    garbage, speed, unit = line.split(' ')
    multiplier = 1000 if unit == 'Mbit/s' else 1
    return int(float(speed)*multiplier)


def convert_host(line):
    m = re.match(r"Hosted by (.+) \((.+)\) \[(.+)\]\: (.+) ms", line)
    hostname, location, distance, latency = m.groups()
    return {
                'hostname': hostname,
                'location': location,
                'distance': distance,
                'latency': latency
            }


def from_stream(stream):
    data = []
    output = {}
    depth = 0
    for line in stream:
        line = line.strip()
        if line.find('-----Start:') == 0:
            if depth > 0:
                warning('Incomplete data!', output)
                depth = 0
            depth += 1
            output = { 'start': convert_datetime(line) }
        elif line.find('-----End:') == 0:
            output['end'] = convert_datetime(line)
            if 'start' not in output or \
                'down' not in output or \
                'up'   not in output or \
                'hostname' not in output or \
                'location' not in output or \
                'distance' not in output or \
                'latency'  not in output:
                warning('Incomplete data!', output)
            if 'start' in output and \
                'down' in output:
                data.append(output)
            depth = 0
            output = {}
        elif line.find('Download:') == 0:
            output['down'] = convert_speed(line)
        elif line.find('Upload:') == 0:
            output['up'] = convert_speed(line)
        elif line.find('Hosted by') == 0:
            output.update(convert_host(line))
    return data


if __name__ == "__main__":

    db = sql.mysqlite()
    data = from_stream(sys.stdin)
    records = 0
    for datum in data:
        try:
            db.insert('speedtest', datum)
            records += 1
        except sql.IntegrityError:
            warning('Error: Insert: ID already exists: {}'
                    .format(datum['start']))
    db.commit()
    print('{} records inserted!'.format(records))<commit_msg>Added command line file specification options.<commit_after>from __future__ import print_function
import re
import sys
import sqlite3
from pprint import pprint

import mysqlite as sql


TABLE_NAME = 'speedtest'


def warning(*objs):
    print("WARNING: ", *objs, file=sys.stderr)


def convert_datetime(line):
    garbage, datetime = line.split(' ')
    return datetime


def convert_speed(line):
    garbage, speed, unit = line.split(' ')
    multiplier = 1000 if unit == 'Mbit/s' else 1
    return int(float(speed)*multiplier)


def convert_host(line):
    m = re.match(r"Hosted by (.+) \((.+)\) \[(.+)\]\: (.+) ms", line)
    hostname, location, distance, latency = m.groups()
    return {
                'hostname': hostname,
                'location': location,
                'distance': distance,
                'latency': latency
            }


def from_stream(stream):
    data = []
    output = {}
    depth = 0
    for line in stream:
        line = line.strip()
        if line.find('-----Start:') == 0:
            if depth > 0:
                warning('Incomplete data!', output)
                depth = 0
            depth += 1
            output = { 'start': convert_datetime(line) }
        elif line.find('-----End:') == 0:
            output['end'] = convert_datetime(line)
            if 'start' not in output or \
                'down' not in output or \
                'up'   not in output or \
                'hostname' not in output or \
                'location' not in output or \
                'distance' not in output or \
                'latency'  not in output:
                warning('Incomplete data!', output)
            if 'start' in output and \
                'down' in output:
                data.append(output)
            depth = 0
            output = {}
        elif line.find('Download:') == 0:
            output['down'] = convert_speed(line)
        elif line.find('Upload:') == 0:
            output['up'] = convert_speed(line)
        elif line.find('Hosted by') == 0:
            output.update(convert_host(line))
    return data


def convert_log_to_sqlite(sqlite_filename, log_stream):
    db = sql.mysqlite(sqlite_filename)
    data = from_stream(log_stream)
    records = 0
    for datum in data:
        try:
            db.insert(TABLE_NAME, datum)
            records += 1
        except sql.IntegrityError:
            warning('Error: Insert: ID already exists: {}'
                    .format(datum['start']))
    db.commit()
    print('{} records inserted!'.format(records))


if __name__ == "__main__":
    if len(sys.argv) > 1:
        sqlite_filename = sys.argv[1]
    else:
        sqlite_filename = None
    if len(sys.argv) > 2:
        log_stream = open(sys.argv[2])
    else:
        log_stream = sys.stdin
    convert_log_to_sqlite(sqlite_filename, log_stream)
{'fraction_non_alphanumeric': 0.1264799849652321, 'fraction_numerical': 0.007517383950385266, 'mean_word_length': 1.8158730158730159, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6186920', 'n_tokens_mistral': 1608, 'n_tokens_neox': 1518, 'n_words': 421}