<commit_before>"""
beam_aligner
Author: <NAME>
"""


class Alignment(object):
    def __init__(self, final_node, source_seq, target_seq):
        """
        Construct a new alignment from the given parameters.

        :param final_node:
        :param source_seq:
        :param target_seq:
        :return:
        :rtype: Alignment
        """
        self.__nodes = final_node.flatten()
        self.__nodes.reverse()

        self.source_seq = source_seq
        self.target_seq = target_seq
        self.cost = final_node.cost

    def size(self):
        return len(self.__nodes)

    def node_at(self, align_x):
        return self.__nodes[align_x]

    def errors(self):
        """
        Get all the errors in the alignment
        :return:
        """
        def is_error(node):
            return (node.align_type == AlignmentType.SUB or
                    node.align_type == AlignmentType.INS or
                    node.align_type == AlignmentType.DEL)

        return filter(is_error, self.__nodes)

    def errors_n(self):
        """
        Get the total number of errors in the alignment; i.e. the total number of substitutions, insertions, and
        deletions

        :return: total errors
        :rtype: int
        """
        return len(self.errors())

    def correct_n(self):
        """
        Get the total number of matches in the alignment.

        :return:
        :rtype: int
        """
        return len(filter(lambda n: n.align_type == AlignmentType.MATCH, self.__nodes))

    def wer(self):
        """
        Get the Word Error Rate for this alignment. Naturally, you can do this even if the alignment
        doesn't consist of words.

        :return: word error rate
        :rtype: float
        """
        err_n = float(self.errors_n())
        return err_n / (err_n + self.correct_n())

    def as_tuples(self):
        return [(node.source_token(self.source_seq), node.target_token(self.target_seq)) for node in self.__nodes]

    def __str__(self):
        return ("size={} len(source)={}, len(target)={}, cost={}, WER={}"
                .format(self.size(), len(self.source_seq), len(self.target_seq), self.cost, self.wer())
                )

    @staticmethod
    def __normalize_for_logging(s):
        return s.replace("\n", "\\n").replace(' ', "' '")

    def pretty_print(self):
        pretty = self.__str__() + "\n"
        for node in self.__nodes:
            pretty += node.pretty_print(self.source_seq, self.target_seq) + "\n"
        return pretty


class AlignmentType:
    def __init__(self):
        raise AssertionError("Cannot instantiate this 'enum' type")

    START = "START"
    MATCH = "MATCH"
    SUB = "SUB"
    INS = "INS"
    DEL = "DEL"


class AlignmentNode(object):
    def __init__(self, align_type, previous, source_pos, target_pos, cost):
        """

        :rtype: AlignmentNode
        """
        self.align_type = align_type
        self.previous = previous
        self.cost = cost
        self.sourcePos = source_pos
        self.targetPos = target_pos

    def flatten(self):
        """
        Flatten the linked nodes into a list.
        :return:
        :rtype: list
        """
        flat = []
        current = self
        while current != Aligner.START_NODE:
            flat.append(current)
            current = current.previous
        return flat

    def is_source_end(self, source):
        return self.sourcePos >= len(source) - 1

    def is_target_end(self, target):
        return self.targetPos >= len(target) - 1

    def source_token(self, source_seq, empty=''):
        if self.align_type == AlignmentType.INS:
            return empty
        return source_seq[self.sourcePos]

    def target_token(self, target_seq, empty=''):
        if self.align_type == AlignmentType.DEL:
            return empty
        return target_seq[self.targetPos]

    def pretty_print(self, source_seq, target_seq):
        return ("{:<30}{:^10}{:>30}   {:<5}"
                .format(self.source_token(source_seq), self.align_type, self.target_token(target_seq), self.cost))

    def __eq__(self, other):
        return (
            self.previous == other.previous and
            self.cost == other.cost and<commit_msg>minor to flatten()<commit_after>"""
beam_aligner
Author: <NAME>
"""


class Alignment(object):
    def __init__(self, final_node, source_seq, target_seq):
        """
        Construct a new alignment from the given parameters.

        :param final_node:
        :param source_seq:
        :param target_seq:
        :return:
        :rtype: Alignment
        """
        self.__nodes = final_node.flatten()
        # self.__nodes.reverse()

        self.source_seq = source_seq
        self.target_seq = target_seq
        self.cost = final_node.cost

    def size(self):
        return len(self.__nodes)

    def node_at(self, align_x):
        return self.__nodes[align_x]

    def errors(self):
        """
        Get all the errors in the alignment
        :return:
        """
        def is_error(node):
            return (node.align_type == AlignmentType.SUB or
                    node.align_type == AlignmentType.INS or
                    node.align_type == AlignmentType.DEL)

        return filter(is_error, self.__nodes)

    def errors_n(self):
        """
        Get the total number of errors in the alignment; i.e. the total number of substitutions, insertions, and
        deletions

        :return: total errors
        :rtype: int
        """
        return len(self.errors())

    def correct_n(self):
        """
        Get the total number of matches in the alignment.

        :return:
        :rtype: int
        """
        return len(filter(lambda n: n.align_type == AlignmentType.MATCH, self.__nodes))

    def wer(self):
        """
        Get the Word Error Rate for this alignment. Naturally, you can do this even if the alignment
        doesn't consist of words.

        :return: word error rate
        :rtype: float
        """
        err_n = float(self.errors_n())
        return err_n / (err_n + self.correct_n())

    def as_tuples(self):
        return [(node.source_token(self.source_seq), node.target_token(self.target_seq)) for node in self.__nodes]

    def __str__(self):
        return ("size={} len(source)={}, len(target)={}, cost={}, WER={}"
                .format(self.size(), len(self.source_seq), len(self.target_seq), self.cost, self.wer())
                )

    @staticmethod
    def __normalize_for_logging(s):
        return s.replace("\n", "\\n").replace(' ', "' '")

    def pretty_print(self):
        pretty = self.__str__() + "\n"
        for node in self.__nodes:
            pretty += node.pretty_print(self.source_seq, self.target_seq) + "\n"
        return pretty


class AlignmentType:
    def __init__(self):
        raise AssertionError("Cannot instantiate this 'enum' type")

    START = "START"
    MATCH = "MATCH"
    SUB = "SUB"
    INS = "INS"
    DEL = "DEL"


class AlignmentNode(object):
    def __init__(self, align_type, previous, source_pos, target_pos, cost):
        """

        :rtype: AlignmentNode
        """
        self.align_type = align_type
        self.previous = previous
        self.cost = cost
        self.sourcePos = source_pos
        self.targetPos = target_pos

    def flatten(self):
        """
        Flatten the linked nodes into a list.
        :return:
        :rtype: list
        """
        flat = []
        current = self
        while current != Aligner.START_NODE:
            flat.insert(0, current)
            # flat.append(current)
            current = current.previous
        return flat

    def is_source_end(self, source):
        return self.sourcePos >= len(source) - 1

    def is_target_end(self, target):
        return self.targetPos >= len(target) - 1

    def source_token(self, source_seq, empty=''):
        if self.align_type == AlignmentType.INS:
            return empty
        return source_seq[self.sourcePos]

    def target_token(self, target_seq, empty=''):
        if self.align_type == AlignmentType.DEL:
            return empty
        return target_seq[self.targetPos]

    def pretty_print(self, source_seq, target_seq):
        return ("{:<30}{:^10}{:>30}   {:<5}"
                .format(self.source_token(source_seq), self.align_type, self.target_token(target_seq), self.cost))

    def __eq__(self, other):
        return (
            self.previous == other.previous and
            self.cost == other.cost and
{'fraction_non_alphanumeric': 0.11331844363465382, 'fraction_numerical': 0.0022334548019278242, 'mean_word_length': 2.0893246187363834, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7124388', 'n_tokens_mistral': 2570, 'n_tokens_neox': 2450, 'n_words': 635}