<commit_before># -*- coding: utf-8 -*-
# Natural Language Toolkit: GDFA word alignment symmetrization
#
# Copyright (C) 2001-2013 NLTK Project
# Authors: <NAME>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

import codecs
from collections import defaultdict

def grow_diag_final_and(srclen, trglen, e2f, f2e):
    """
    This module symmetrisatizes the source-to-target and target-to-source
    word alignment output and produces, aka. GDFA algorithm (Koehn, 2005).
    
    Step 1 - Find the intersection of the bidirectional alignment.
    
    Step 2 - Search for additional neighbor alignment points to be added, 
             given these criteria 
            (i) neighbor alignments points are not in the intersection and 
            (ii) neighbor alignments are in the union.
            
    Step 3 - Adds all other alignment points thats not in the intersection,
             not in the neighboring alignments that met the criteria but in the 
             original foward/backward alignment outputs.
    
    [in]:
    *e2f* is the forward word alignment outputs from source-to-target language.
    *f2e* is the backward word alignment outputs from target-to-source langauge.
    *srclen* is the number of tokens in the source language.
    *trglen* is the number of tokens in the target language.
    (Note: both *e2f* and *f2e* is in pharaoh output format) 
    
    [out]:
    *alignment* is a symmetrized alignment points from GDFA algorithm.
    
    >>> forward = '0-0 2-1 9-2 21-3 10-4 7-5 11-6 9-7 12-8 1-9 3-10 4-11 17-12 17-13 25-14 13-15 24-16 11-17 28-18'
    >>> backward = '0-0 1-9 2-9 3-10 4-11 5-12 6-6 7-5 8-6 9-7 10-4 11-6 12-8 13-12 15-12 17-13 18-13 19-12 20-13 21-3 22-12 23-14 24-17 25-15 26-17 27-18 28-18'<commit_msg>fixed typo error in function description.<commit_after># -*- coding: utf-8 -*-
# Natural Language Toolkit: GDFA word alignment symmetrization
#
# Copyright (C) 2001-2013 NLTK Project
# Authors: <NAME>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

import codecs
from collections import defaultdict

def grow_diag_final_and(srclen, trglen, e2f, f2e):
    """
    This module symmetrisatizes the source-to-target and target-to-source
    word alignment output and produces, aka. GDFA algorithm (Koehn, 2005).
    
    Step 1 - Find the intersection of the bidirectional alignment.
    
    Step 2 - Search for additional neighbor alignment points to be added, 
             given these criteria 
            (i) neighbor alignments points are not in the intersection and 
            (ii) neighbor alignments are in the union.
            
    Step 3 - Adds all other alignment points thats not in the intersection,
             not in the neighboring alignments that met the criteria but in the 
             original forward/backward alignment outputs.
    
    [in]:
    *e2f* is the forward word alignment outputs from source-to-target language.
    *f2e* is the backward word alignment outputs from target-to-source langauge.
    *srclen* is the number of tokens in the source language.
    *trglen* is the number of tokens in the target language.
    (Note: both *e2f* and *f2e* is in pharaoh output format) 
    
    [out]:
    *alignment* is a symmetrized alignment points from GDFA algorithm.
    
    >>> forward = '0-0 2-1 9-2 21-3 10-4 7-5 11-6 9-7 12-8 1-9 3-10 4-11 17-12 17-13 25-14 13-15 24-16 11-17 28-18'
    >>> backward = '0-0 1-9 2-9 3-10 4-11 5-12 6-6 7-5 8-6 9-7 10-4 11-6 12-8 13-12 15-12 17-13 18-13 19-12 20-13 21-3 22-12 23-14 24-17 25-15 26-17 27-18 28-18'
{'fraction_non_alphanumeric': 0.08796425579447081, 'fraction_numerical': 0.09327003630270873, 'mean_word_length': 2.592778335005015, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3697272', 'n_tokens_mistral': 1357, 'n_tokens_neox': 1104, 'n_words': 462}