<commit_before>from typing import List

import numpy

class IndexedInstance:
    """
    An indexed data instance, which is a list of word indices, coupled with a label, and possibly
    an instance index.  An IndexedInstance is created from an Instance using a DataIndexer, and the
    indices here have no recoverable meaning without the DataIndexer.

    For example, we might have the following instance:
        Instance('Jamie is nice, Holly is mean', True, 25).
    After being converted into an IndexedInstance, we might have the following:
        IndexedInstance([1, 6, 7, 1, 6, 8], True, 25).
    This would mean that "Jamie" and "Holly" were OOV to the DataIndexer, and the other words were
    given indices.
    """
    def __init__(self, word_indices: List[int], label: bool, index: int=None):
        self.word_indices = word_indices
        self.label = label
        self.index = index

    def get_lengths(self) -> List[int]:
        """
        This simple IndexedInstance only has one padding dimension: word_indices.
        """
        return [len(self.word_indices)]

    def pad(self, max_lengths: List[int]):
        """
        Pads (or truncates) self.word_indices to be of length max_lengths[0].  See comment on
        self.get_lengths() for why this is a list instead of an int.
        """
        desired_length = max_lengths[0]
        padded_word_indices = [0] * desired_length
        indices_length = min(len(self.word_indices), desired_length)
        if indices_length != 0:
            padded_word_indices[-indices_length:] = self.word_indices[-indices_length:]
        self.word_indices = padded_word_indices

    def as_training_data(self):
        word_array = numpy.asarray(self.word_indices, dtype='int32')
        label = numpy.zeros((2))
        if self.label is True:
            label[1] = 1
        elif self.label is False:
            label[0] = 1
        else:
            raise RuntimeError("Cannot make training data out of instances without labels!")
        return word_array, label


class IndexedLogicalFormInstance(IndexedInstance):<commit_msg>Improved comment on instance padding<commit_after>from typing import List

import numpy

class IndexedInstance:
    """
    An indexed data instance, which is a list of word indices, coupled with a label, and possibly
    an instance index.  An IndexedInstance is created from an Instance using a DataIndexer, and the
    indices here have no recoverable meaning without the DataIndexer.

    For example, we might have the following instance:
        Instance('Jamie is nice, Holly is mean', True, 25).
    After being converted into an IndexedInstance, we might have the following:
        IndexedInstance([1, 6, 7, 1, 6, 8], True, 25).
    This would mean that "Jamie" and "Holly" were OOV to the DataIndexer, and the other words were
    given indices.
    """
    def __init__(self, word_indices: List[int], label: bool, index: int=None):
        self.word_indices = word_indices
        self.label = label
        self.index = index

    def get_lengths(self) -> List[int]:
        """
        This simple IndexedInstance only has one padding dimension: word_indices.
        """
        return [len(self.word_indices)]

    def pad(self, max_lengths: List[int]):
        """
        Pads (or truncates) self.word_indices to be of length max_lengths[0].  See comment on
        self.get_lengths() for why max_lengths is a list instead of an int.

        If we need to truncate self.word_indices, we do it from the _right_, not the left.  This is
        important for cases that are questions, with long set ups.  We at least want to get the
        question encoded, which is always at the end, even if we've lost much of the question set
        up.
        """
        desired_length = max_lengths[0]
        padded_word_indices = [0] * desired_length
        indices_length = min(len(self.word_indices), desired_length)
        if indices_length != 0:
            padded_word_indices[-indices_length:] = self.word_indices[-indices_length:]
        self.word_indices = padded_word_indices

    def as_training_data(self):
        word_array = numpy.asarray(self.word_indices, dtype='int32')
        label = numpy.zeros((2))
        if self.label is True:
            label[1] = 1
        elif self.label is False:
            label[0] = 1
        else:
            raise RuntimeError("Cannot make training data out of instances without labels!")
        return word_array, label


class IndexedLogicalFormInstance(IndexedInstance):
{'fraction_non_alphanumeric': 0.07876787678767877, 'fraction_numerical': 0.009240924092409241, 'mean_word_length': 2.769485903814262, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2549924', 'n_tokens_mistral': 1302, 'n_tokens_neox': 1251, 'n_words': 471}