<commit_before>#!/usr/bin/env python

import matplotlib.cm as cm
import matplotlib.pyplot as plt
import itertools
from operator import itemgetter
import datetime
from collections import defaultdict
import numpy as np
import StringIO
import json

# Dictionary with project names and the corresponding number of data
# points which should be removed from the end.
TRIMMINGS = {
    "0044_AD035EACXX": 1,
    "0050_AD037LACXX": 3,
    "0054_AC00H7ABXX": 1,
    "0055_BB06RCABXX": 2,
    "0157_B80NNMABXX": 1,
    "0158_A80NNNABXX": 1,
    "0175_B814PKABXX": 1,
    "0177_AB02VJABXX": 1,
    "0178_BB02UMABXX": 2,
    "0192_BB051UABXX": 2,
    "0201_BB06GNABXX": 1,
    "0202_AB0AA5ABXX": 1,
    "0203_BB039NABXX": 1,
    "0204_AB028LABXX": 2,
    "0205_BB047AABXX": 2,
    "0208_AB0B0KABXX": 1,
    "0209_BB0ADVABXX": 1,
    "0212_AB0B10ABXX": 1,
    "0214_B80LRFABXX": 2,
    "0215_B80J7PABXX": 1,
    "0219_BB0B29ABXX": 1,
    "0241_AC00RAABXX": 1,
    "0242_BB08CAABXX": 1,
    "0249_A816HRABXX": 1,
    "0250_B816J0ABXX": 2,
    "0251_A8158LABXX": 3,
    "0254_AB03BEABXX": 3,
    "0255_A81BF6ABXX": 2,
    "0255_BB04BAABXX": 2,
    "0256_B81CL7ABXX": 2,
    "0257_A81BHDABXX": 3,
    "0258_B819KCABXX": 3,
    "0261_A81BFKABXX": 2,
    "0262_B819K6ABXX": 2,
    "0264_B81BELABXX": 1,
    "0268_BB02J7ABXX": 2,
    "0269_AB02EVABXX": 1,
    "0270_BD035NACXX": 2,
    "0271_AB03T6ABXX": 1,
    "0273_AC00HFABXX": 2,
    "0274_BB028HABXX": 1,
    "0277_AB02J6ABXX": 1,
    "0278_BB0B0HABXX": 2,
    "0279_AC03E3ABXX": 1,
    "0280_BB0AC9ABXX": 1,
    "0281_AD0DYGABXX": 1,
    "0282_BC04B5ABXX": 2,
    "0283_AB06PBABXX": 2,
    "0284_BB06CNABXX": 2
}


def convert_human_readable_to_numbers(file_size):
    """In data, convert letters which indicate file sizes to actual numbers.
    """
    dpref = file_size[-1]
    if dpref not in ["K", "M", "G", "T"]:
        return 0

    file_size = file_size[:-1]
    if dpref == 'G':
        file_size = int(float(file_size) * 1024 * 1024 * 1024)
    elif dpref == 'T':
        file_size = int(float(file_size) * 1024 * 1024 * 1024 * 1024)
    elif dpref == 'M':
        file_size = int(float(file_size) * 1024 * 1024)
    elif dpref == 'K':
        file_size = int(float(file_size) * 1024)

    return file_size


def get_formatted_data():
    """Returns the data, processed a bit to be more regular.
    """
    # This of course need to be pointed to wherever the log file is available.
    with open("/Users/valentinesvensson/Documents/sizegraphing/datasets_sizes.log") as data_f:
        all_data = data_f.read()

    data_lines = all_data.splitlines()
    data = [line.split("\t") for line in data_lines]
    for line in data:
        line[-1] = line[-1].split("/")[-1]

    data = [line for line in data if "." not in line[-1]]
    sizes = [int(convert_human_readable_to_numbers(line[0])) for line in data]
    data = [line[-1].split("_") for line in data]

    for i in range(len(sizes)):
        data[i].insert(0, sizes[i])

    data = [line for line in data if len(line) == 5]

    for i in range(len(data)):
        data[i][3:5] = ["_".join(data[i][3:5])]
        data[i][1] = datetime.datetime.strptime(data[i][1], "%y%m%d")

    for line in data:
        line[2] = line[2].upper()
        line[3] = line[3].upper()

    return data


def trim_data(grouped_data, trim=True):
    """Trims away the file sizes where preprocessing has occured.
    """
    trimmings = TRIMMINGS

    sorted_data = []

    for group in grouped_data:
        sizes = []
        for line in group:
            sizes.append(line[0])
        sizes = sorted(sizes)
        if trim:
            remove = trimmings.get(line[-1], None)
            if remove is not None:
                sizes = sizes[:-remove]

        sorted_data.append([sizes, [line[1]], line[2], line[3]])

    return sorted_data


def get_list_of_machines():
    """Returns a list of machine serial numbers found in the logs.
    """
    data = get_formatted_data()
    data = [list(g) for k, g in itertools.groupby(data, itemgetter(3))]
    data = trim_data(data)
    machines = set([])
    for proj in data:
        machines.add(proj[2])<commit_msg>Added a function for getting the old log data in the format we want it in the database<commit_after>#!/usr/bin/env python

import matplotlib.cm as cm
import matplotlib.pyplot as plt
import itertools
from operator import itemgetter
import datetime
from collections import defaultdict
import numpy as np
import StringIO
import json

# Dictionary with project names and the corresponding number of data
# points which should be removed from the end.
TRIMMINGS = {
    "0044_AD035EACXX": 1,
    "0050_AD037LACXX": 3,
    "0054_AC00H7ABXX": 1,
    "0055_BB06RCABXX": 2,
    "0157_B80NNMABXX": 1,
    "0158_A80NNNABXX": 1,
    "0175_B814PKABXX": 1,
    "0177_AB02VJABXX": 1,
    "0178_BB02UMABXX": 2,
    "0192_BB051UABXX": 2,
    "0201_BB06GNABXX": 1,
    "0202_AB0AA5ABXX": 1,
    "0203_BB039NABXX": 1,
    "0204_AB028LABXX": 2,
    "0205_BB047AABXX": 2,
    "0208_AB0B0KABXX": 1,
    "0209_BB0ADVABXX": 1,
    "0212_AB0B10ABXX": 1,
    "0214_B80LRFABXX": 2,
    "0215_B80J7PABXX": 1,
    "0219_BB0B29ABXX": 1,
    "0241_AC00RAABXX": 1,
    "0242_BB08CAABXX": 1,
    "0249_A816HRABXX": 1,
    "0250_B816J0ABXX": 2,
    "0251_A8158LABXX": 3,
    "0254_AB03BEABXX": 3,
    "0255_A81BF6ABXX": 2,
    "0255_BB04BAABXX": 2,
    "0256_B81CL7ABXX": 2,
    "0257_A81BHDABXX": 3,
    "0258_B819KCABXX": 3,
    "0261_A81BFKABXX": 2,
    "0262_B819K6ABXX": 2,
    "0264_B81BELABXX": 1,
    "0268_BB02J7ABXX": 2,
    "0269_AB02EVABXX": 1,
    "0270_BD035NACXX": 2,
    "0271_AB03T6ABXX": 1,
    "0273_AC00HFABXX": 2,
    "0274_BB028HABXX": 1,
    "0277_AB02J6ABXX": 1,
    "0278_BB0B0HABXX": 2,
    "0279_AC03E3ABXX": 1,
    "0280_BB0AC9ABXX": 1,
    "0281_AD0DYGABXX": 1,
    "0282_BC04B5ABXX": 2,
    "0283_AB06PBABXX": 2,
    "0284_BB06CNABXX": 2
}


def convert_human_readable_to_numbers(file_size):
    """In data, convert letters which indicate file sizes to actual numbers.
    """
    dpref = file_size[-1]
    if dpref not in ["K", "M", "G", "T"]:
        return 0

    file_size = file_size[:-1]
    if dpref == 'G':
        file_size = int(float(file_size) * 1024 * 1024 * 1024)
    elif dpref == 'T':
        file_size = int(float(file_size) * 1024 * 1024 * 1024 * 1024)
    elif dpref == 'M':
        file_size = int(float(file_size) * 1024 * 1024)
    elif dpref == 'K':
        file_size = int(float(file_size) * 1024)

    return file_size


def get_formatted_data():
    """Returns the data, processed a bit to be more regular.
    """
    # This of course need to be pointed to wherever the log file is available.
    with open("/Users/valentinesvensson/Documents/sizegraphing/datasets_sizes.log") as data_f:
        all_data = data_f.read()

    data_lines = all_data.splitlines()
    data = [line.split("\t") for line in data_lines]
    for line in data:
        line[-1] = line[-1].split("/")[-1]

    data = [line for line in data if "." not in line[-1]]
    sizes = [int(convert_human_readable_to_numbers(line[0])) for line in data]
    data = [line[-1].split("_") for line in data]

    for i in range(len(sizes)):
        data[i].insert(0, sizes[i])

    data = [line for line in data if len(line) == 5]

    for i in range(len(data)):
        data[i][3:5] = ["_".join(data[i][3:5])]
        data[i][1] = datetime.datetime.strptime(data[i][1], "%y%m%d")

    for line in data:
        line[2] = line[2].upper()
        line[3] = line[3].upper()

    return data


def get_dirs_with_sizes_and_dates():
    """Returns a list of dicts as we want them in the status data base.
    """
    # This of course need to be pointed to wherever the log file is available.
    with open("/Users/valentinesvensson/Documents/sizegraphing/datasets_sizes.log") as data_f:
        all_data = data_f.read()

    data_lines = all_data.splitlines()
    data = [line.split("\t") for line in data_lines]

    for line in data:
        line[0] = int(convert_human_readable_to_numbers(line[0]))
        date_string = line[1].split("/")[-1].split("_")[0]
        line[1] = line[1].split("/")[-1]
        try:
            line.append(datetime.datetime.strptime(date_string, "%y%m%d"))
        except:
            pass

    data = [line for line in data if len(line) == 3]

    string_picker = lambda l: l[1]
    data = sorted(data, key=string_picker)

    data = [list(g) for k, g in itertools.groupby(data, string_picker)]

    size_picker = lambda l: l[0]
    for dir_name_list in data:
        dir_name_list = sorted(dir_name_list, key=size_picker)
        for i, entry in enumerate(dir_name_list):
            entry[-1] = entry[-1] + datetime.timedelta(days=i)

    flat_data = []
    for dir_name_list in data:
        flat_data.extend(dir_name_list)

    date_picker = lambda l: l[-1]
    flat_data = sorted(flat_data, key=date_picker)
    data = [list(g) for k, g in itertools.groupby(flat_data, date_picker)]

    log_dicts = []

    for date in data:
        log_dict = {"time": date_picker(date[0]).isoformat()}
        for date_data in date:
            log_dict[string_picker(date_data)] = size_picker(date_data)

        log_dicts.append(log_dict)

    return log_dicts


def trim_data(grouped_data, trim=True):
    """Trims away the file sizes where preprocessing has occured.
    """
    trimmings = TRIMMINGS

    sorted_data = []

    for group in grouped_data:
        sizes = []
        for line in group:
            sizes.append(line[0])
        sizes = sorted(sizes)
        if trim:
            remove = trimmings.get(line[-1], None)
            if remove is not None:
                sizes = sizes[:-remove]

        sorted_data.append([sizes, [line[1]], line[2], line[3]])

    return sorted_data


def get_list_of_machines():
    """Returns a list of machine serial numbers found in the logs.
    """
    data = get_formatted_data()
    data = [list(g) for k, g in itertools.groupby(data, itemgetter(3))]
    data = trim_data(data)
    machines = set([])
    for proj in data:
        machines.add(proj[2])
{'fraction_non_alphanumeric': 0.1316966504323626, 'fraction_numerical': 0.08836099791273233, 'mean_word_length': 2.9289340101522843, 'pattern_counts': {'":': 99, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3263578', 'n_tokens_mistral': 4446, 'n_tokens_neox': 3900, 'n_words': 926}