<commit_before>import gzip
from math import ceil
import pickle
import random
import numpy as np
import os
import sys
from load_sequences import run
from load_sequences import get_rec

datasets_dir = 'media/datasets/'


def one_hot(x, n):
    if type(x) == list:
        x = np.array(x)
    x = x.flatten()
    o_h = np.zeros((len(x), n))
    o_h[np.arange(len(x)), x] = 1
    return o_h


def seq_to_bits(vec):
    bits_vector = []
    for i, c in enumerate(vec):
        if c == "A":
            bits_vector += [1, 0, 0, 0]
        elif c == "T":
            bits_vector += [0, 1, 0, 0]
        elif c == "C":
            bits_vector += [0, 0, 1, 0]
        elif c == "G":
            bits_vector += [0, 0, 0, 1]
        else:
            bits_vector += [1, 1, 1, 1]
    return bits_vector

    # for i, c in enumerate(vec):
    #     if c == "A":
    #         bits_vector.append(0)
    #     elif c == "T":
    #         bits_vector.append(0.33)
    #     elif c == "C":
    #         bits_vector.append(0.66)
    #     elif c == "G":
    #         bits_vector.append(1)
    #     else:
    #         bits_vector.append(0.5)
    # return bits_vector



def load_seqs(ids):
    return [get_rec(x).seq._data for x in ids]  # nalozi sekvence


def seq_load(ntrain=50000, ntest=10000, onehot=True, seed=random.randint(0, sys.maxint)):

    seq_len = 100

    # nastavi seed
    random.seed(seed)

    # preveri ce obstaja datoteka s podatki o taxonomy (samo id-ji in labeli), ce ne obstaja pozeni run in shrani

    dir = "media"
    try:
        print "loading data and labels..."
        data = pickle.load(open(dir + "/data-ids.pkl.gz", "rb"))
        labels = pickle.load(open(dir + "/labels.pkl.gz", "rb"))
    except IOError:
        print "data and labels not found...\ngenerating data and labels..."
        data, labels = run()
        print "saving data and labels..."
        pickle.dump(data, open(dir + "/data-ids.pkl.gz", "wb"), -1)
        pickle.dump(labels, open(dir + "/labels.pkl.gz", "wb"), -1)

    print "getting sequences..."
    data = load_seqs(data)

    try:
        print "loading train and test data for seed %d..." % seed
        trX = pickle.load(open(dir + "/train-data-seed_%d.pkl.gz" % seed, "rb"))
        trY = pickle.load(open(dir + "/train-labels-seed_%d.pkl.gz" % seed, "rb"))
        teX = pickle.load(open(dir + "/t10k-data-seed_%d.pkl.gz" % seed, "rb"))
        teY = pickle.load(open(dir + "/t10k-labels-seed_%d.pkl.gz" % seed, "rb"))
    except IOError:  # , FileNotFoundError:
        print "train and test data not found for seed %d..." % seed
        print "generating train and test data..."
        number_of_classes = len(set(labels))
        # train_examples_per_class = int(ceil(ntrain / float(number_of_classes)))
        # test_examples_per_class = int(ceil(ntest / float(number_of_classes)))
        # popravi da se bo v trX vstavljalo zaporedoma za vsak label
        train_examples_per_class = ntrain / number_of_classes
        test_examples_per_class = ntest / number_of_classes
        examples_per_class = train_examples_per_class + test_examples_per_class

        trX = []
        trY = []
        teX = []
        teY = []

        temp_count = 0

        while temp_count < (ntrain + ntest):
            for label in set(labels):
                first = labels.index(label)
                last = len(labels) - labels[::-1].index(label)
                print "number of examples in class: %d" % (last - first)

                sum_lengths = sum(len(s) for s in data[first:last])
                print "sum lengths of genomes: %d" % sum_lengths

                vir_idx     = random.choice(range(first, last))  # nakljucno izberi virus iz razreda
                sample_idx  = random.choice(range(0, (len(data[vir_idx])) - seq_len - 1))  # nakljucno vzorci virus

                if temp_count < ntrain:
                    trX.append(seq_to_bits(data[vir_idx][sample_idx:sample_idx + seq_len]))
                    trY.append(label)
                else:
                    teX.append(seq_to_bits(data[vir_idx][sample_idx:sample_idx + seq_len]))
                    teY.append(label)

                temp_count += 1

        print "saving train and test data for seed %d..." % seed
        pickle.dump(trX, open(dir + "/train-data-seed_%d.pkl.gz" % seed, "wb"), -1)
        pickle.dump(trY, open(dir + "/train-labels-seed_%d.pkl.gz" % seed, "wb"), -1)
        pickle.dump(teX, open(dir + "/t10k-data-seed_%d.pkl.gz" % seed, "wb"), -1)
        pickle.dump(teY, open(dir + "/t10k-labels-seed_%d.pkl.gz" % seed, "wb"), -1)
        print "saving done"

    number_of_classes = len(set(labels))

    print len(trX)
    print len(teX)

    if onehot:
        trY = one_hot(trY, number_of_classes)
        teY = one_hot(teY, number_of_classes)<commit_msg>test podatki samo do ntest<commit_after>import gzip
from math import ceil
import pickle
import random
import numpy as np
import os
import sys
from load_sequences import run
from load_sequences import get_rec

datasets_dir = 'media/datasets/'


def one_hot(x, n):
    if type(x) == list:
        x = np.array(x)
    x = x.flatten()
    o_h = np.zeros((len(x), n))
    o_h[np.arange(len(x)), x] = 1
    return o_h


def seq_to_bits(vec):
    bits_vector = []
    for i, c in enumerate(vec):
        if c == "A":
            bits_vector += [1, 0, 0, 0]
        elif c == "T":
            bits_vector += [0, 1, 0, 0]
        elif c == "C":
            bits_vector += [0, 0, 1, 0]
        elif c == "G":
            bits_vector += [0, 0, 0, 1]
        else:
            bits_vector += [1, 1, 1, 1]
    return bits_vector

    # for i, c in enumerate(vec):
    #     if c == "A":
    #         bits_vector.append(0)
    #     elif c == "T":
    #         bits_vector.append(0.33)
    #     elif c == "C":
    #         bits_vector.append(0.66)
    #     elif c == "G":
    #         bits_vector.append(1)
    #     else:
    #         bits_vector.append(0.5)
    # return bits_vector



def load_seqs(ids):
    return [get_rec(x).seq._data for x in ids]  # nalozi sekvence


def seq_load(ntrain=50000, ntest=10000, onehot=True, seed=random.randint(0, sys.maxint)):

    seq_len = 100

    # nastavi seed
    random.seed(seed)

    # preveri ce obstaja datoteka s podatki o taxonomy (samo id-ji in labeli), ce ne obstaja pozeni run in shrani

    dir = "media"
    try:
        print "loading data and labels..."
        data = pickle.load(open(dir + "/data-ids.pkl.gz", "rb"))
        labels = pickle.load(open(dir + "/labels.pkl.gz", "rb"))
    except IOError:
        print "data and labels not found...\ngenerating data and labels..."
        data, labels = run()
        print "saving data and labels..."
        pickle.dump(data, open(dir + "/data-ids.pkl.gz", "wb"), -1)
        pickle.dump(labels, open(dir + "/labels.pkl.gz", "wb"), -1)

    print "getting sequences..."
    data = load_seqs(data)

    try:
        print "loading train and test data for seed %d..." % seed
        trX = pickle.load(open(dir + "/train-data-seed_%d.pkl.gz" % seed, "rb"))
        trY = pickle.load(open(dir + "/train-labels-seed_%d.pkl.gz" % seed, "rb"))
        teX = pickle.load(open(dir + "/t10k-data-seed_%d.pkl.gz" % seed, "rb"))
        teY = pickle.load(open(dir + "/t10k-labels-seed_%d.pkl.gz" % seed, "rb"))
    except IOError:  # , FileNotFoundError:
        print "train and test data not found for seed %d..." % seed
        print "generating train and test data..."
        number_of_classes = len(set(labels))
        # train_examples_per_class = int(ceil(ntrain / float(number_of_classes)))
        # test_examples_per_class = int(ceil(ntest / float(number_of_classes)))
        # popravi da se bo v trX vstavljalo zaporedoma za vsak label
        train_examples_per_class = ntrain / number_of_classes
        test_examples_per_class = ntest / number_of_classes
        examples_per_class = train_examples_per_class + test_examples_per_class

        trX = []
        trY = []
        teX = []
        teY = []

        temp_count = 0

        while temp_count < (ntrain + ntest):
            for label in set(labels):
                first = labels.index(label)
                last = len(labels) - labels[::-1].index(label)
                print "number of examples in class: %d" % (last - first)

                sum_lengths = sum(len(s) for s in data[first:last])
                print "sum lengths of genomes: %d" % sum_lengths

                vir_idx     = random.choice(range(first, last))  # nakljucno izberi virus iz razreda
                sample_idx  = random.choice(range(0, (len(data[vir_idx])) - seq_len - 1))  # nakljucno vzorci virus

                if temp_count < ntrain:
                    trX.append(seq_to_bits(data[vir_idx][sample_idx:sample_idx + seq_len]))
                    trY.append(label)
                else:
                    if temp_count - ntrain > ntest:
                        break
                    teX.append(seq_to_bits(data[vir_idx][sample_idx:sample_idx + seq_len]))
                    teY.append(label)

                temp_count += 1

        print "saving train and test data for seed %d..." % seed
        pickle.dump(trX, open(dir + "/train-data-seed_%d.pkl.gz" % seed, "wb"), -1)
        pickle.dump(trY, open(dir + "/train-labels-seed_%d.pkl.gz" % seed, "wb"), -1)
        pickle.dump(teX, open(dir + "/t10k-data-seed_%d.pkl.gz" % seed, "wb"), -1)
        pickle.dump(teY, open(dir + "/t10k-labels-seed_%d.pkl.gz" % seed, "wb"), -1)
        print "saving done"

    number_of_classes = len(set(labels))

    print len(trX)
    print len(teX)

    if onehot:
        trY = one_hot(trY, number_of_classes)
        teY = one_hot(teY, number_of_classes)
{'fraction_non_alphanumeric': 0.13545202098117864, 'fraction_numerical': 0.013164661112825259, 'mean_word_length': 2.082091917591125, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1227138', 'n_tokens_mistral': 3533, 'n_tokens_neox': 3357, 'n_words': 824}