<commit_before>import sys
from mwt import *
from calculate import *

multiary = sys.argv[1]
input_file_name = sys.argv[2]

input_file = (input_file_name, "r")
input_string = input_file.readline().strip()

if __name__ == "__main__":
    num_of_layers = get_number_of_layers(multiary)
    layers = create_layers(num_of_layers)<commit_msg>Finished layers<commit_after>import sys
from mwt import *
from calculate import *

# read arguments
multiary = int(sys.argv[1])
input_file_name = sys.argv[2]

# open input file and put it into a string
input_file = open(input_file_name, "r")
input_string = input_file.readline().strip()

if __name__ == "__main__":
    # calculating needed information
    num_of_layers = base = get_number_of_layers(multiary)
    layers = create_layers(num_of_layers)

    # filling the layers, preparing data for MWT creation
    dictionary = {}
    for sign in input_string:
        if sign in dictionary:
            counter = 0
            for s in dictionary[sign]:
                layers[counter].append(s)
                counter += 1
        else:
            dictionary[sign] = change_base(ord(sign), multiary, base)
            counter = 0
            for s in dictionary[sign]:
                layers[counter].append(s)
                counter += 1

    # MWT creation

{'fraction_non_alphanumeric': 0.09230769230769231, 'fraction_numerical': 0.006153846153846154, 'mean_word_length': 2.930513595166163, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6560774', 'n_tokens_mistral': 401, 'n_tokens_neox': 390, 'n_words': 105}