<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: <NAME> <sennrich [AT] cl.uzh.ch>

# This program handles the combination of Moses phrase tables, either through
# linear interpolation of the phrase translation probabilities/lexical weights,
# or through a recomputation based on the (weighted) combined counts.
#
# It also supports an automatic search for weights that minimize the cross-entropy
# between the model and a tuning set of word/phrase alignments.

# for usage information, run
# python tmcombine.py -h
# you can also check the docstrings of Combine_TMs() for more information and find some example commands in the function test()


# Some general things to note:
#  - Different combination algorithms require different statistics. To be on the safe side, use the options `-phrase-word-alignment` and `-write-lexical-counts` when training models.
#  - The script assumes that phrase tables are sorted (to allow incremental, more memory-friendly processing). sort with LC_ALL=C.
#  - Some configurations require additional statistics that are loaded in memory (lexical tables; complete list of target phrases). If memory consumption is a problem, use the option --lowmem (slightly slower and writes temporary files to disk), or consider pruning your phrase table before combining (e.g. using Johnson et al. 2007).
#  - The script can read/write gzipped files, but the Python implementation is slow. You're better off unzipping the files on the command line and working with the unzipped files.
#  - The cross-entropy estimation assumes that phrase tables contain true probability distributions (i.e. a probability mass of 1 for each conditional probability distribution). If this is not true, the results are skewed.
#  - Unknown phrase pairs are not considered for the cross-entropy estimation. A comparison of models with different vocabularies may be misleading.
#  - Don't directly compare cross-entropies obtained from a combination with different modes. Depending on how some corner cases are treated, linear interpolation does not distribute full probability mass and thus shows higher (i.e. worse) cross-entropies.


from __future__ import division, unicode_literals
import sys
import os
import gzip
import argparse
import copy
import re
from math import log, exp
from collections import defaultdict
from operator import mul
from tempfile import NamedTemporaryFile
from subprocess import Popen
try:
    from itertools import izip
except:
    izip = zip

try:
    from lxml import etree as ET
except:
    import xml.etree.cElementTree as ET

try:
    from scipy.optimize.lbfgsb import fmin_l_bfgs_b
    optimizer = 'l-bfgs'
except:
    optimizer = 'hillclimb'

class Moses():
    """Moses interface for loading/writing models
    to support other phrase table formats, subclass this and overwrite the relevant functions
    """
    
    def __init__(self,models,number_of_features):

        self.number_of_features = number_of_features
        self.models = models

        #example item (assuming mode=='counts' and one feature): phrase_pairs['the house']['das haus'] = [[[10,100]],['0-0 1-1']]
        self.phrase_pairs = defaultdict(lambda: defaultdict(lambda: [[[0]*len(self.models) for i in range(self.number_of_features)],[]]))
        self.phrase_source = defaultdict(lambda: [0]*len(self.models))
        self.phrase_target =  defaultdict(lambda: [0]*len(self.models))
        
        self.reordering_pairs = defaultdict(lambda: defaultdict(lambda: [[0]*len(self.models) for i in range(self.number_of_features)]))
        
        self.word_pairs_e2f = defaultdict(lambda: defaultdict(lambda: [0]*len(self.models)))
        self.word_pairs_f2e = defaultdict(lambda: defaultdict(lambda: [0]*len(self.models)))
        self.word_source = defaultdict(lambda: [0]*len(self.models))
        self.word_target = defaultdict(lambda: [0]*len(self.models))
        
        self.require_alignment = False
        

    def open_table(self,model,table,mode='r'):
        """define which paths to open for lexical tables and phrase tables.
            we assume canonical Moses structure, but feel free to overwrite this
        """
        
        if table == 'reordering-table':
            table = 'reordering-table.wbe-msd-bidirectional-fe'
        
        filename = os.path.join(model,'model',table)
        fileobj = handle_file(filename,'open',mode)
        return fileobj
        
        
    def load_phrase_features(self,line,priority,i,mode='interpolate',store='pairs',filter_by=None,filter_by_src=None,filter_by_target=None,inverted=False,flags=None):
        """take single phrase table line and store probablities in internal data structure"""
        
        src = line[0]
        target = line[1]
        
        if inverted:
            src,target = target,src
        
        if (store == 'all' or store == 'pairs') and (priority < 10 or (src in self.phrase_pairs and target in self.phrase_pairs[src])) and not (filter_by and not (src in filter_by and target in filter_by[src])):
            
                self.store_info(src,target,line)
                
                scores = line[2].split()
                if len(scores) <self.number_of_features:
                    sys.stderr.write('Error: model only has {0} features. Expected {1}.\n'.format(len(scores),self.number_of_features))
                    exit()
                    
                scores = scores[:self.number_of_features]
                model_probabilities = map(float,scores)
                phrase_probabilities = self.phrase_pairs[src][target][0]
                
                if mode == 'counts' and not priority == 2: #priority 2 is MAP
                    try:
                        target_count,src_count = map(float,line[-1].split())
                    except:
                        sys.stderr.write(str(line)+'\n')
                        sys.stderr.write('Counts are missing. Maybe your phrase table is from an older Moses version that doesn\'t store counts?\n')
                        return
                        
                    i_e2f = flags['i_e2f']
                    i_f2e = flags['i_f2e']
                    model_probabilities[i_e2f] *= target_count
                    model_probabilities[i_f2e] *= src_count
                
                for j,p in enumerate(model_probabilities):
                    phrase_probabilities[j][i] = p
                
        # mark that the src/target phrase has been seen.
        # needed for re-normalization during linear interpolation
        if (store == 'all' or store == 'source') and not (filter_by_src and not src in filter_by_src):
            if mode == 'counts' and not priority == 2: #priority 2 is MAP
                try:
                    self.phrase_source[src][i] = float(line[-1].split()[1])
                except:
                    sys.stderr.write(str(line)+'\n')
                    sys.stderr.write('Counts are missing. Maybe your phrase table is from an older Moses version that doesn\'t store counts?\n')
                    return
            else:
                self.phrase_source[src][i] = 1
                
        if (store == 'all' or store == 'target') and not (filter_by_target and not target in filter_by_target):
            if mode == 'counts' and not priority == 2: #priority 2 is MAP
                try:
                    self.phrase_target[target][i] = float(line[-1].split()[0])
                except:
                    sys.stderr.write(str(line)+'\n')
                    sys.stderr.write('Counts are missing. Maybe your phrase table is from an older Moses version that doesn\'t store counts?\n')
                    return
            else:
                self.phrase_target[target][i] = 1

<commit_msg>fix regression from having all 3 counts in phrase table.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: <NAME> <sennrich [AT] cl.uzh.ch>

# This program handles the combination of Moses phrase tables, either through
# linear interpolation of the phrase translation probabilities/lexical weights,
# or through a recomputation based on the (weighted) combined counts.
#
# It also supports an automatic search for weights that minimize the cross-entropy
# between the model and a tuning set of word/phrase alignments.

# for usage information, run
# python tmcombine.py -h
# you can also check the docstrings of Combine_TMs() for more information and find some example commands in the function test()


# Some general things to note:
#  - Different combination algorithms require different statistics. To be on the safe side, use the options `-phrase-word-alignment` and `-write-lexical-counts` when training models.
#  - The script assumes that phrase tables are sorted (to allow incremental, more memory-friendly processing). sort with LC_ALL=C.
#  - Some configurations require additional statistics that are loaded in memory (lexical tables; complete list of target phrases). If memory consumption is a problem, use the option --lowmem (slightly slower and writes temporary files to disk), or consider pruning your phrase table before combining (e.g. using Johnson et al. 2007).
#  - The script can read/write gzipped files, but the Python implementation is slow. You're better off unzipping the files on the command line and working with the unzipped files.
#  - The cross-entropy estimation assumes that phrase tables contain true probability distributions (i.e. a probability mass of 1 for each conditional probability distribution). If this is not true, the results are skewed.
#  - Unknown phrase pairs are not considered for the cross-entropy estimation. A comparison of models with different vocabularies may be misleading.
#  - Don't directly compare cross-entropies obtained from a combination with different modes. Depending on how some corner cases are treated, linear interpolation does not distribute full probability mass and thus shows higher (i.e. worse) cross-entropies.


from __future__ import division, unicode_literals
import sys
import os
import gzip
import argparse
import copy
import re
from math import log, exp
from collections import defaultdict
from operator import mul
from tempfile import NamedTemporaryFile
from subprocess import Popen
try:
    from itertools import izip
except:
    izip = zip

try:
    from lxml import etree as ET
except:
    import xml.etree.cElementTree as ET

try:
    from scipy.optimize.lbfgsb import fmin_l_bfgs_b
    optimizer = 'l-bfgs'
except:
    optimizer = 'hillclimb'

class Moses():
    """Moses interface for loading/writing models
    to support other phrase table formats, subclass this and overwrite the relevant functions
    """
    
    def __init__(self,models,number_of_features):

        self.number_of_features = number_of_features
        self.models = models

        #example item (assuming mode=='counts' and one feature): phrase_pairs['the house']['das haus'] = [[[10,100]],['0-0 1-1']]
        self.phrase_pairs = defaultdict(lambda: defaultdict(lambda: [[[0]*len(self.models) for i in range(self.number_of_features)],[]]))
        self.phrase_source = defaultdict(lambda: [0]*len(self.models))
        self.phrase_target =  defaultdict(lambda: [0]*len(self.models))
        
        self.reordering_pairs = defaultdict(lambda: defaultdict(lambda: [[0]*len(self.models) for i in range(self.number_of_features)]))
        
        self.word_pairs_e2f = defaultdict(lambda: defaultdict(lambda: [0]*len(self.models)))
        self.word_pairs_f2e = defaultdict(lambda: defaultdict(lambda: [0]*len(self.models)))
        self.word_source = defaultdict(lambda: [0]*len(self.models))
        self.word_target = defaultdict(lambda: [0]*len(self.models))
        
        self.require_alignment = False
        

    def open_table(self,model,table,mode='r'):
        """define which paths to open for lexical tables and phrase tables.
            we assume canonical Moses structure, but feel free to overwrite this
        """
        
        if table == 'reordering-table':
            table = 'reordering-table.wbe-msd-bidirectional-fe'
        
        filename = os.path.join(model,'model',table)
        fileobj = handle_file(filename,'open',mode)
        return fileobj
        
        
    def load_phrase_features(self,line,priority,i,mode='interpolate',store='pairs',filter_by=None,filter_by_src=None,filter_by_target=None,inverted=False,flags=None):
        """take single phrase table line and store probablities in internal data structure"""
        
        src = line[0]
        target = line[1]
        
        if inverted:
            src,target = target,src
        
        if (store == 'all' or store == 'pairs') and (priority < 10 or (src in self.phrase_pairs and target in self.phrase_pairs[src])) and not (filter_by and not (src in filter_by and target in filter_by[src])):
            
                self.store_info(src,target,line)
                
                scores = line[2].split()
                if len(scores) <self.number_of_features:
                    sys.stderr.write('Error: model only has {0} features. Expected {1}.\n'.format(len(scores),self.number_of_features))
                    exit()
                    
                scores = scores[:self.number_of_features]
                model_probabilities = map(float,scores)
                phrase_probabilities = self.phrase_pairs[src][target][0]
                
                if mode == 'counts' and not priority == 2: #priority 2 is MAP
                    try:
                        counts = map(float,line[-1].split())
                        try:
                            target_count,src_count,joint_count = counts
                            joint_count_e2f = joint_count
                            joint_count_f2e = joint_count
                        except ValueError:
                            # possibly old-style phrase table with 2 counts in last column, or phrase table produced by tmcombine
                            # note: since each feature has different weight vector, we may have two different phrase pair frequencies
                            target_count,src_count = counts
                            i_e2f = flags['i_e2f']
                            i_f2e = flags['i_f2e']
                            joint_count_e2f = model_probabilities[i_e2f] * target_count
                            joint_count_f2e = model_probabilities[i_f2e] * src_count
                    except:
                        sys.stderr.write(str(b" ||| ".join(line))+b'\n')
                        sys.stderr.write('ERROR: counts are missing or misformatted. Maybe your phrase table is from an older Moses version that doesn\'t store counts?\n')
                        raise
                    
                    i_e2f = flags['i_e2f']
                    i_f2e = flags['i_f2e']
                    model_probabilities[i_e2f] = joint_count_e2f
                    model_probabilities[i_f2e] = joint_count_f2e
                        
                for j,p in enumerate(model_probabilities):
                    phrase_probabilities[j][i] = p
                
        # mark that the src/target phrase has been seen.
        # needed for re-normalization during linear interpolation
        if (store == 'all' or store == 'source') and not (filter_by_src and not src in filter_by_src):
            if mode == 'counts' and not priority == 2: #priority 2 is MAP
                try:
                    self.phrase_source[src][i] = float(line[-1].split()[1])
                except:
                    sys.stderr.write(str(line)+'\n')
                    sys.stderr.write('Counts are missing. Maybe your phrase table is from an older Moses version that doesn\'t store counts?\n')
                    return
            else:
                self.phrase_source[src][i] = 1
                
        if (store == 'all' or store == 'target') and not (filter_by_target and not target in filter_by_target):
            if mode == 'counts' and not priority == 2: #priority 2 is MAP
                try:
                    self.phrase_target[target][i] = float(line[-1].split()[0])
                except:
                    sys.stderr.write(str(line)+'\n')
                    sys.stderr.write('Counts are missing. Maybe your phrase table is from an older Moses version that doesn\'t store counts?\n')
                    return
            else:
                self.phrase_target[target][i] = 1


{'fraction_non_alphanumeric': 0.08674552623597209, 'fraction_numerical': 0.0071580224446466485, 'mean_word_length': 2.3998762631470405, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6176209', 'n_tokens_mistral': 4488, 'n_tokens_neox': 4279, 'n_words': 1480}