<commit_before># -*- coding: utf-8 -*-
import re
from datetime import datetime
from operator import itemgetter

MIN_TIME = datetime.strptime('1998-04-30 21:30:17', '%Y-%m-%d %H:%M:%S')
MAX_TIME = datetime.strptime('1998-07-26 21:59:55', '%Y-%m-%d %H:%M:%S')
BUFF_COUNT = 10000
REGULAR = re.compile(r'\[(\S+)\s(\S+)\]')


def pre_process(in_file, out_file=None, sub='all', interval=0):
    result = {}
    result = read_from_file(in_file, result, sub, interval * 60)
    ordered = sorted(result.items(), key=itemgetter(0))
    hint_order = sorted(result.items(), key=itemgetter(1), reverse=True)
    output(out_file, ordered, hint_order)


def big_file_read_test(in_file, interval=0, flag=False):
    dict_ = {}
    begin = datetime.now()
    with open(in_file, 'r') as in_file, open('./log/read_test.log', 'w') as log:
        index = 0
        for line in in_file:
            index += 1
            if flag:
                if interval == 0:
                    key = data_process(line)
                else:
                    key = data_process(line) / interval
                val = dict_.get(key)
                if val:
                    val += 1
                else:
                    val = 1
                dict_[key] = val
        duration = (datetime.now() - begin).total_seconds()
        log.write("read and parse cost seconds %s" % duration)
    print (datetime.now() - begin).total_seconds()


def read_from_file(file_, dict_, sub, interval):
    begin = datetime.now()
    log_file = './log/pre_proc_%s.log' % sub
    with open(file_, 'r') as in_file, open(log_file, 'w') as log:
        index = 0
        for line in in_file:
            index += 1
            if interval == 0:
                key = data_process(line)
            else:
                key = data_process(line) / interval
            val = dict_.get(key)
            if val:
                val += 1
            else:
                val = 1
            dict_[key] = val
        duration = (datetime.now() - begin).total_seconds()
        log.write("read and parse cost seconds %s" % duration)
    print (datetime.now() - begin).total_seconds()
    return dict_


def parse_time(time_str):
    to_parse = time_str.split(' ')[0]<commit_msg>remove file read test for preprocess<commit_after># -*- coding: utf-8 -*-
import re
from datetime import datetime
from operator import itemgetter

MIN_TIME = datetime.strptime('1998-04-30 21:30:17', '%Y-%m-%d %H:%M:%S')
MAX_TIME = datetime.strptime('1998-07-26 21:59:55', '%Y-%m-%d %H:%M:%S')
BUFF_COUNT = 10000
REGULAR = re.compile(r'\[(\S+)\s(\S+)\]')


def pre_process(in_file, out_file=None, sub='all', interval=0):
    result = {}
    result = read_from_file(in_file, result, sub, interval * 60)
    ordered = sorted(result.items(), key=itemgetter(0))
    hint_order = sorted(result.items(), key=itemgetter(1), reverse=True)
    output(out_file, ordered, hint_order)


def read_from_file(file_, dict_, sub, interval):
    begin = datetime.now()
    log_file = './log/pre_proc_%s.log' % sub
    with open(file_, 'r') as in_file, open(log_file, 'w') as log:
        index = 0
        for line in in_file:
            index += 1
            if interval == 0:
                key = data_process(line)
            else:
                key = data_process(line) / interval
            val = dict_.get(key)
            if val:
                val += 1
            else:
                val = 1
            dict_[key] = val
        duration = (datetime.now() - begin).total_seconds()
        log.write("read and parse cost seconds %s" % duration)
    print (datetime.now() - begin).total_seconds()
    return dict_


def parse_time(time_str):
    to_parse = time_str.split(' ')[0]
{'fraction_non_alphanumeric': 0.1418554476806904, 'fraction_numerical': 0.025889967637540454, 'mean_word_length': 2.159284497444634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5645199', 'n_tokens_mistral': 1268, 'n_tokens_neox': 1175, 'n_words': 295}