<commit_before># Import dependencies
import os, logging, json
from datasource import TrainingInstance as tri
import numpy as np
from utils import feature_extractor as fe
import pickle
import random

def generate_label_sequences(labels, n_instances, l_range=(1, 30)):
    '''
    Generates a given number of label sequences
    :param labels: a list of available unique labels
    :param n_instances: number of instances (sequences) to generate
    :param range: A tuple that holds the min and max length of sequences
    :return: list of sequences and list of their corresponding lengths
    '''
    seqs = []
    seq_lengths = []

    for i in np.arange(l_range[0], l_range[1]+1):
        logging.info('Generating sequences of length ' + str(i))
        for j in range(n_instances):
            if j%20 == 0:
                logging.info('Generating ' + str(j) + 'th sequence of length '
                             + str(i))
            seqs.append(random.sample(labels, i))
            seq_lengths.append(i)
    return seqs, seq_lengths

def generate_data_sequences(codebook, labele_seqs, l_range=(1, 30)):
    '''
    Generates a given number of data sequences
    :param codebook: A codebook (dict obj) that holds all the labels as keys
    and all the corresponding data instances as values
    :param labele_seqs: Sequence of labels for which data sequences needs to
    be generated
    :param range: A tuple that holds the min and max length of sequences
    :return:
    '''

    seqs = []
    t_seqs = []

    for j, label_seq in enumerate(labele_seqs) :
        if j % 20 == 0 :
            logging.info('Generating ' + str(j) + 'th data sequence of length '
                         + str(len(label_seq)))
        d_seq_buf = None
        for lbl in label_seq:
            if d_seq_buf is None:
                d_seq_buf = np.array(random.sample(codebook[lbl], 1))
            else:
                rand_lbl_sample = random.sample(codebook[lbl], 1)
                d_seq_buf = np.concatenate((d_seq_buf, rand_lbl_sample), axis=1)

        seqs.append(d_seq_buf)
        t_seqs.append(np.transpose(d_seq_buf))

    return seqs, t_seqs


def scaleData(data, scaler):
    '''
    Method to scale the sensor data as a preprocessing step
    :param data: (list) List of all the training instance objects
    :return: data: (list) List of a ll the scaled training data objects
    '''<commit_msg>Added data sequence generator and sequence dumping module. Need to test both of them. Minor fixes<commit_after># Import dependencies
import os, logging, json
from datasource import TrainingInstance as tri
import numpy as np
from utils import feature_extractor as fe
import pickle
import random

def generate_label_sequences(labels, n_instances, l_range=(1, 30)):
    '''
    Generates a given number of label sequences
    :param labels: a list of available unique labels
    :param n_instances: number of instances (sequences) to generate
    :param range: A tuple that holds the min and max length of sequences
    :return: list of sequences and list of their corresponding lengths
    '''
    seqs = []
    seq_lengths = []

    for i in np.arange(l_range[0], l_range[1]+1):
        logging.info('Generating sequences of length ' + str(i))
        for j in range(n_instances):
            if j%20 == 0:
                logging.info('Generating ' + str(j) + 'th sequence of length '
                             + str(i))
            seqs.append(random.sample(labels, i))
            seq_lengths.append(i)
    return seqs, seq_lengths

def generate_data_sequences(codebook, labele_seqs, l_range=(1, 30)):
    '''
    Generates a given number of data sequences
    :param codebook: A codebook (dict obj) that holds all the labels as keys
    and all the corresponding data instances as values
    :param labele_seqs: Sequence of labels for which data sequences needs to
    be generated
    :param range: A tuple that holds the min and max length of sequences
    :return:
    '''

    seqs = []
    t_seqs = []

    for j, label_seq in enumerate(labele_seqs) :
        if j % 20 == 0 :
            logging.info('Generating ' + str(j) + 'th data sequence of length '
                         + str(len(label_seq)))
        d_seq_buf = None
        for lbl in label_seq:
            if d_seq_buf is None:
                sampled_data = random.sample(codebook[lbl], 1)
                d_seq_buf = np.array(sampled_data)
            else:
                rand_lbl_sample = random.sample(codebook[lbl], 1)
                d_seq_buf = np.concatenate((d_seq_buf, rand_lbl_sample), axis=1)

        seqs.append(d_seq_buf)
        t_seqs.append(np.transpose(d_seq_buf))

    return seqs, t_seqs


def scaleData(data, scaler):
    '''
    Method to scale the sensor data as a preprocessing step
    :param data: (list) List of all the training instance objects
    :return: data: (list) List of a ll the scaled training data objects
    '''
{'fraction_non_alphanumeric': 0.08205024311183144, 'fraction_numerical': 0.007293354943273906, 'mean_word_length': 2.5905454545454547, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1319475', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1431, 'n_words': 517}