<commit_before>import argparse
from fuel.datasets import H5PYDataset
import h5py
from numpy import array, save
import re
from os.path import exists

PROGRESS_STEP = 10000

OUT_FILE_NAME = './data/lk.hdf5'
RAW_DATA_OUT_NAME = './data/lk_cleaned.data'
ALPHABET_FILE = './data/lk_ix2char.npy'

comment_pattern = re.compile('//|/\*')


def get_raw_data(file_path):
    with open(file_path) as f:
        c = '#'  # dummy value, not used
        trigger = ''
        read = 1
        raw = ''
        while c:
            try:
                c = f.read(1)
            except UnicodeDecodeError:
                pass
            else:
                if read:
                    if c == '/':
                        c += f.read(1)
                        if comment_pattern.match(c):
                            read = 0
                            trigger = c
                    else:
                        raw += c
                else:
                    if trigger == '//' and c == '\n':
                        read = 1
                        raw += c
                    elif trigger == '/*' and c == '*':
                        c += f.read(1)
                        if c == '*/':
                            read = 1
    return raw

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--setsize', type=float, default=.95, help='Determines the proportion of training data, value between 0 and 1.')
    parser.add_argument('-f', '--file', type=str, default='', help='The file to be converted.')


    args = parser.parse_args()

    training_size = args.setsize
    test_size = 1 - training_size

    if not args.file:
        with open(RAW_DATA_OUT_NAME) as f:
            raw = f.read()
    else:
        raw = get_raw_data(args.file)
        with open(RAW_DATA_OUT_NAME, 'w') as f:
            f.write(raw)

    seq = []
    seqs = []
    char2int = {}
    ix2char = {}
    ix = 0
    di = 0  # debug
    upper_limit = int((len(raw) * 15000000)/500000000)
    print(upper_limit)  #10111676

    batch_size = 1000

    for b in 

    for i in range(0, len(raw), fixed_len):
        line = raw[i:i+fixed_len]
        for c in line:
            if c in char2int:
                seq.append(char2int[c])
            else:
                char2int[c] = ix
                seq.append(ix)
                ix += 1
        seqs.append(seq)
        seq = []

    data = array(seqs)

    split_n = int(training_size * data.shape[0])

    # write hdf5

    f = h5py.File(name=OUT_FILE_NAME, mode='w')
    character_seqs = f.create_dataset("character_seqs", data.shape, dtype='int32')
    character_seqs[...] = data

    split_dict = {"train": {"character_seqs": (0, split_n)},
                  "valid": {"character_seqs": (split_n, data.shape[0])}}
    f.attrs["split"] = H5PYDataset.create_split_array(split_dict)<commit_msg>prepare is work in progress<commit_after>import argparse
from fuel.datasets import H5PYDataset
import h5py
from numpy import array, save
import re
from os.path import exists

PROGRESS_STEP = 10000

OUT_FILE_NAME = './data/lk.hdf5'
RAW_DATA_OUT_NAME = './data/lk_cleaned.data'
ALPHABET_FILE = './data/lk_ix2char.npy'

comment_pattern = re.compile('//|/\*')


def get_raw_data(file_path):
    with open(file_path) as f:
        c = '#'  # dummy value, not used
        trigger = ''
        read = 1
        raw = ''
        while c:
            try:
                c = f.read(1)
            except UnicodeDecodeError:
                pass
            else:
                if read:
                    if c == '/':
                        c += f.read(1)
                        if comment_pattern.match(c):
                            read = 0
                            trigger = c
                    else:
                        raw += c
                else:
                    if trigger == '//' and c == '\n':
                        read = 1
                        raw += c
                    elif trigger == '/*' and c == '*':
                        c += f.read(1)
                        if c == '*/':
                            read = 1
    return raw

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--setsize', type=float, default=.95, help='Determines the proportion of training data, value between 0 and 1.')
    parser.add_argument('-f', '--file', type=str, default='', help='The file to be converted.')


    args = parser.parse_args()

    training_size = args.setsize
    test_size = 1 - training_size

    if not args.file:
        with open(RAW_DATA_OUT_NAME) as f:
            raw = f.read()
    else:
        raw = get_raw_data(args.file)
        with open(RAW_DATA_OUT_NAME, 'w') as f:
            f.write(raw)

    seq = []
    seqs = []
    char2int = {}
    ix2char = {}
    ix = 0
    di = 0  # debug
    upper_limit = int((len(raw) * 15000000)/500000000)
    print(upper_limit)  #10111676

    batch_size = 1000

    seqs = []
    p_old = 0

    for b in range(0, upper_limit-batch_size+1, batch_size):
        p = b+batch_size
        while raw[p] != '}':
            p -= 1
        seqs.append(raw[p_old:p+1])
        if not seqs[-1]:
            raise ValueError('Empty sequence added!')
        p_old = p

    print(*[len(s) for s in seqs])
    exit()

    for i in range(0, len(raw), fixed_len):
        line = raw[i:i+fixed_len]
        for c in line:
            if c in char2int:
                seq.append(char2int[c])
            else:
                char2int[c] = ix
                seq.append(ix)
                ix += 1
        seqs.append(seq)
        seq = []

    data = array(seqs)

    split_n = int(training_size * data.shape[0])

    # write hdf5

    f = h5py.File(name=OUT_FILE_NAME, mode='w')
    character_seqs = f.create_dataset("character_seqs", data.shape, dtype='int32')
    character_seqs[...] = data

    split_dict = {"train": {"character_seqs": (0, split_n)},
                  "valid": {"character_seqs": (split_n, data.shape[0])}}
    f.attrs["split"] = H5PYDataset.create_split_array(split_dict)
{'fraction_non_alphanumeric': 0.123377690159356, 'fraction_numerical': 0.022999835715459174, 'mean_word_length': 1.5240464344941957, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2702386', 'n_tokens_mistral': 2068, 'n_tokens_neox': 1886, 'n_words': 461}