<commit_before>from keras_wrapper.dataset import Dataset, saveDataset, loadDataset
import logging

logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] %(message)s', datefmt='%d/%m/%Y %H:%M:%S')


def update_dataset_from_file(ds, text_filename, params, splits=list('val'), remove_outputs=False):
    """
    Updates the dataset instance from a text file according to the given params.
    Used for sampling

    :param ds: Dataset instance
    :param text_filename: Source language sentences
    :param params: Parameters for building the dataset
    :param splits: Splits to sample
    :return: Dataset object with the processed data
    """
    for split in splits:
        if remove_outputs:
            ds.removeOutput(split,
                            type='text',
                            id=params['OUTPUTS_IDS_DATASET'][0])

        # INPUT DATA
        ds.setInput(text_filename,
                    split,
                    type='text',
                    id=params['INPUTS_IDS_DATASET'][0],
                    pad_on_batch=params['PAD_ON_BATCH'],
                    tokenization=params['TOKENIZATION_METHOD'],
                    build_vocabulary=False,
                    fill=params['FILL'],
                    max_text_len=params['MAX_INPUT_TEXT_LEN'],
                    max_words=params['INPUT_VOCABULARY_SIZE'],
                    min_occ=params['MIN_OCCURRENCES_VOCAB'],
                    overwrite_split=True)

        ds.setInput(None,
                    split,
                    type='ghost',
                    id=params['INPUTS_IDS_DATASET'][-1],
                    required=False,
                    overwrite_split=True)

        if params['ALIGN_FROM_RAW']:
            ds.setRawInput(text_filename,
                           split,
                           type='file-name',
                           id='raw_' + params['INPUTS_IDS_DATASET'][0],
                           overwrite_split=True)

    return ds


def build_dataset(params):
    """
    Builds (or loads) a Dataset instance.
    :param params: Parameters specifying Dataset options<commit_msg>Optional output_text_filename in update_dataset_from_file<commit_after>from keras_wrapper.dataset import Dataset, saveDataset, loadDataset
import logging

logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] %(message)s', datefmt='%d/%m/%Y %H:%M:%S')


def update_dataset_from_file(ds, input_text_filename, params, splits=list('val'), output_text_filename=None, remove_outputs=False):
    """
    Updates the dataset instance from a text file according to the given params.
    Used for sampling

    :param ds: Dataset instance
    :param input_text_filename: Source language sentences
    :param params: Parameters for building the dataset
    :param splits: Splits to sample
    :return: Dataset object with the processed data
    """
    for split in splits:
        if remove_outputs:
            ds.removeOutput(split,
                            type='text',
                            id=params['OUTPUTS_IDS_DATASET'][0])
        else:
            ds.setOutput(output_text_filename,
                         split,
                         type='text',
                         id=params['OUTPUTS_IDS_DATASET'][0],
                         tokenization=params['TOKENIZATION_METHOD'],
                         build_vocabulary=False,
                         pad_on_batch=params['PAD_ON_BATCH'],
                         sample_weights=params['SAMPLE_WEIGHTS'],
                         max_text_len=params['MAX_OUTPUT_TEXT_LEN'],
                         max_words=params['OUTPUT_VOCABULARY_SIZE'],
                         min_occ=params['MIN_OCCURRENCES_VOCAB'],
                         overwrite_split=True)


        # INPUT DATA
        ds.setInput(input_text_filename,
                    split,
                    type='text',
                    id=params['INPUTS_IDS_DATASET'][0],
                    pad_on_batch=params['PAD_ON_BATCH'],
                    tokenization=params['TOKENIZATION_METHOD'],
                    build_vocabulary=False,
                    fill=params['FILL'],
                    max_text_len=params['MAX_INPUT_TEXT_LEN'],
                    max_words=params['INPUT_VOCABULARY_SIZE'],
                    min_occ=params['MIN_OCCURRENCES_VOCAB'],
                    overwrite_split=True)

        ds.setInput(None,
                    split,
                    type='ghost',
                    id=params['INPUTS_IDS_DATASET'][-1],
                    required=False,
                    overwrite_split=True)

        if params['ALIGN_FROM_RAW']:
            ds.setRawInput(input_text_filename,
                           split,
                           type='file-name',
                           id='raw_' + params['INPUTS_IDS_DATASET'][0],
                           overwrite_split=True)

    return ds


def build_dataset(params):
    """
    Builds (or loads) a Dataset instance.
    :param params: Parameters specifying Dataset options
{'fraction_non_alphanumeric': 0.09136146496815287, 'fraction_numerical': 0.0017914012738853504, 'mean_word_length': 1.7777777777777777, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7559489', 'n_tokens_mistral': 1369, 'n_tokens_neox': 1317, 'n_words': 248}