natural language toolkit api for alignment and translation objects c 20012023 nltk project will zhang wilzzhagmail com guan gui gguistudent unimelb edu au steven bird stevenbird1gmail com tah wei hoon hoon twgmail com url https www nltk org for license information see license txt return an aligned sentence object which encapsulates two sentences along with an alignment between them typically used in machine translation to represent a sentence and its translation from nltk translate import alignedsent alignment algnsent alignedsent klein ist das haus the house is small alignment fromstring 03 12 20 31 algnsent words klein ist das haus algnsent mots the house is small algnsent alignment alignment0 3 1 2 2 0 3 1 from nltk corpus import comtrans printcomtrans alignedsents54 alignedsent weshalb also sollten so why should eu arm printcomtrans alignedsents54 alignment 00 01 10 22 34 35 47 58 63 79 89 910 911 1012 116 126 1313 param words words in the target language sentence type words liststr param mots words in the source language sentence type mots liststr param alignment wordlevel alignments between words and mots each alignment is represented as a 2tuple wordsindex motsindex type alignment alignment return a string representation for this alignedsent rtype str dot representation of the aligned sentence declare node alignment connect the source words connect the target words put it in the same rank ipython magic show svg representation of this alignedsent return a humanreadable string representation for this alignedsent rtype str return the aligned sentence pair reversing the directionality rtype alignedsent a storage class for representing alignment between two sequences s1 s2 in general an alignment is a set of tuples of the form i j representing an alignment between the ith element of s1 and the jth element of s2 tuples are extensible they might contain additional data such as a boolean to indicate sure vs possible alignments from nltk translate import alignment a alignment0 0 0 1 1 2 2 2 a invert alignment0 0 1 0 2 1 2 2 printa invert 00 10 21 22 a0 0 1 0 0 a invert2 2 1 2 2 b alignment0 0 0 1 b issubseta true c alignment fromstring 00 01 b c true read a gizaformatted string and return an alignment object alignment fromstring 00 21 92 213 104 75 alignment0 0 2 1 7 5 9 2 10 4 21 3 type s str param s the positional alignments in giza format rtype alignment return an alignment object corresponding to the string representation s look up the alignments that map from a given index or slice return an alignment object being the inverted mapping work out the range of the mapping from the given positions if no positions are specified compute the range of the entire mapping produce a gizaformatted string representing the alignment produce a gizaformatted string representing the alignment build a list self index such that self indexi is a list of the alignments originating from word i check whether the alignments are legal param numwords the number of source language words type numwords int param nummots the number of target language words type nummots int param alignment alignment to be checked type alignment alignment raise indexerror if alignment falls outside the sentence inmemory store of translations for a given phrase and the log probability of the those translations get the translations for a source language phrase param srcphrase source language phrase of interest type srcphrase tuplestr return a list of target language phrases that are translations of srcphrase ordered in decreasing order of likelihood each list element is a tuple of the target phrase and its log probability rtype listphrasetableentry type srcphrase tuplestr type trgphrase tuplestr param logprob log probability that given srcphrase trgphrase is its translation type logprob float natural language toolkit api for alignment and translation objects c 2001 2023 nltk project will zhang wilzzha gmail com guan gui ggui student unimelb edu au steven bird stevenbird1 gmail com tah wei hoon hoon tw gmail com url https www nltk org for license information see license txt return an aligned sentence object which encapsulates two sentences along with an alignment between them typically used in machine translation to represent a sentence and its translation from nltk translate import alignedsent alignment algnsent alignedsent klein ist das haus the house is small alignment fromstring 0 3 1 2 2 0 3 1 algnsent words klein ist das haus algnsent mots the house is small algnsent alignment alignment 0 3 1 2 2 0 3 1 from nltk corpus import comtrans print comtrans aligned_sents 54 alignedsent weshalb also sollten so why should eu arm print comtrans aligned_sents 54 alignment 0 0 0 1 1 0 2 2 3 4 3 5 4 7 5 8 6 3 7 9 8 9 9 10 9 11 10 12 11 6 12 6 13 13 param words words in the target language sentence type words list str param mots words in the source language sentence type mots list str param alignment word level alignments between words and mots each alignment is represented as a 2 tuple words_index mots_index type alignment alignment return a string representation for this alignedsent rtype str dot representation of the aligned sentence declare node alignment connect the source words connect the target words put it in the same rank ipython magic show svg representation of this alignedsent return a human readable string representation for this alignedsent rtype str return the aligned sentence pair reversing the directionality rtype alignedsent a storage class for representing alignment between two sequences s1 s2 in general an alignment is a set of tuples of the form i j representing an alignment between the i th element of s1 and the j th element of s2 tuples are extensible they might contain additional data such as a boolean to indicate sure vs possible alignments from nltk translate import alignment a alignment 0 0 0 1 1 2 2 2 a invert alignment 0 0 1 0 2 1 2 2 print a invert 0 0 1 0 2 1 2 2 a 0 0 1 0 0 a invert 2 2 1 2 2 b alignment 0 0 0 1 b issubset a true c alignment fromstring 0 0 0 1 b c true read a giza formatted string and return an alignment object alignment fromstring 0 0 2 1 9 2 21 3 10 4 7 5 alignment 0 0 2 1 7 5 9 2 10 4 21 3 type s str param s the positional alignments in giza format rtype alignment return an alignment object corresponding to the string representation s look up the alignments that map from a given index or slice return an alignment object being the inverted mapping work out the range of the mapping from the given positions if no positions are specified compute the range of the entire mapping produce a giza formatted string representing the alignment produce a giza formatted string representing the alignment build a list self _index such that self _index i is a list of the alignments originating from word i check whether the alignments are legal param num_words the number of source language words type num_words int param num_mots the number of target language words type num_mots int param alignment alignment to be checked type alignment alignment raise indexerror if alignment falls outside the sentence in memory store of translations for a given phrase and the log probability of the those translations get the translations for a source language phrase param src_phrase source language phrase of interest type src_phrase tuple str return a list of target language phrases that are translations of src_phrase ordered in decreasing order of likelihood each list element is a tuple of the target phrase and its log probability rtype list phrasetableentry type src_phrase tuple str type trg_phrase tuple str param log_prob log probability that given src_phrase trg_phrase is its translation type log_prob float
import subprocess
from collections import namedtuple


class AlignedSent:
    

    def __init__(self, words, mots, alignment=None):
        self._words = words
        self._mots = mots
        if alignment is None:
            self.alignment = Alignment([])
        else:
            assert type(alignment) is Alignment
            self.alignment = alignment

    @property
    def words(self):
        return self._words

    @property
    def mots(self):
        return self._mots

    def _get_alignment(self):
        return self._alignment

    def _set_alignment(self, alignment):
        _check_alignment(len(self.words), len(self.mots), alignment)
        self._alignment = alignment

    alignment = property(_get_alignment, _set_alignment)

    def __repr__(self):
        
        words = "[%s]" % (", ".join("'%s'" % w for w in self._words))
        mots = "[%s]" % (", ".join("'%s'" % w for w in self._mots))

        return f"AlignedSent({words}, {mots}, {self._alignment!r})"

    def _to_dot(self):
        
        s = "graph align {\n"
        s += "node[shape=plaintext]\n"

        
        for w in self._words:
            s += f'"{w}_source" [label="{w}"] \n'

        for w in self._mots:
            s += f'"{w}_target" [label="{w}"] \n'

        
        for u, v in self._alignment:
            s += f'"{self._words[u]}_source" -- "{self._mots[v]}_target" \n'

        
        for i in range(len(self._words) - 1):
            s += '"{}_source" -- "{}_source" [style=invis]\n'.format(
                self._words[i],
                self._words[i + 1],
            )

        
        for i in range(len(self._mots) - 1):
            s += '"{}_target" -- "{}_target" [style=invis]\n'.format(
                self._mots[i],
                self._mots[i + 1],
            )

        
        s += "{rank = same; %s}\n" % (" ".join('"%s_source"' % w for w in self._words))
        s += "{rank = same; %s}\n" % (" ".join('"%s_target"' % w for w in self._mots))

        s += "}"

        return s

    def _repr_svg_(self):
        
        dot_string = self._to_dot().encode("utf8")
        output_format = "svg"
        try:
            process = subprocess.Popen(
                ["dot", "-T%s" % output_format],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
        except OSError as e:
            raise Exception("Cannot find the dot binary from Graphviz package") from e
        out, err = process.communicate(dot_string)

        return out.decode("utf8")

    def __str__(self):
        
        source = " ".join(self._words)[:20] + "..."
        target = " ".join(self._mots)[:20] + "..."
        return f"<AlignedSent: '{source}' -> '{target}'>"

    def invert(self):
        
        return AlignedSent(self._mots, self._words, self._alignment.invert())


class Alignment(frozenset):
    

    def __new__(cls, pairs):
        self = frozenset.__new__(cls, pairs)
        self._len = max(p[0] for p in self) if self != frozenset([]) else 0
        self._index = None
        return self

    @classmethod
    def fromstring(cls, s):
        

        return Alignment([_giza2pair(a) for a in s.split()])

    def __getitem__(self, key):
        
        if not self._index:
            self._build_index()
        return self._index.__getitem__(key)

    def invert(self):
        
        return Alignment(((p[1], p[0]) + p[2:]) for p in self)

    def range(self, positions=None):
        
        image = set()
        if not self._index:
            self._build_index()
        if not positions:
            positions = list(range(len(self._index)))
        for p in positions:
            image.update(f for _, f in self._index[p])
        return sorted(image)

    def __repr__(self):
        
        return "Alignment(%r)" % sorted(self)

    def __str__(self):
        
        return " ".join("%d-%d" % p[:2] for p in sorted(self))

    def _build_index(self):
        
        self._index = [[] for _ in range(self._len + 1)]
        for p in self:
            self._index[p[0]].append(p)


def _giza2pair(pair_string):
    i, j = pair_string.split("-")
    return int(i), int(j)


def _naacl2pair(pair_string):
    i, j, p = pair_string.split("-")
    return int(i), int(j)


def _check_alignment(num_words, num_mots, alignment):
    

    assert type(alignment) is Alignment

    if not all(0 <= pair[0] < num_words for pair in alignment):
        raise IndexError("Alignment is outside boundary of words")
    if not all(pair[1] is None or 0 <= pair[1] < num_mots for pair in alignment):
        raise IndexError("Alignment is outside boundary of mots")


PhraseTableEntry = namedtuple("PhraseTableEntry", ["trg_phrase", "log_prob"])


class PhraseTable:
    

    def __init__(self):
        self.src_phrases = dict()

    def translations_for(self, src_phrase):
        
        return self.src_phrases[src_phrase]

    def add(self, src_phrase, trg_phrase, log_prob):
        
        entry = PhraseTableEntry(trg_phrase=trg_phrase, log_prob=log_prob)
        if src_phrase not in self.src_phrases:
            self.src_phrases[src_phrase] = []
        self.src_phrases[src_phrase].append(entry)
        self.src_phrases[src_phrase].sort(key=lambda e: e.log_prob, reverse=True)

    def __contains__(self, src_phrase):
        return src_phrase in self.src_phrases