<commit_before>import os

import scripts

__author__ = '<NAME>'


class MosesFeature:
    def __init__(self, classname):
        self.classname = classname

    def get_relpath(self, path):
        path = os.path.abspath(path)
        root = os.path.abspath(scripts.ENGINES_DIR)
        path = path.replace(root, '').lstrip(os.sep)

        path = os.sep.join(path.split(os.sep)[1:])

        return '${ENGINE_PATH}' + path

    def get_iniline(self):
        return None


class LexicalReordering(MosesFeature):
    def __init__(self):
        MosesFeature.__init__(self, 'LexicalReordering')

    def get_iniline(self):
        return 'input-factor=0 output-factor=0 type=hier-mslr-bidirectional-fe-allff'


class Moses:
    injector_section = 'moses'
    injectable_fields = {
        'stack_size': ('search algorithm stack size', int, 5000),
        'cube_pruning_pop_limit': ('pop limit of cube pruning algorithm', int, 5000),
        'distortion_limit': ('distortion limit', int, 6),
    }

    bin_path = os.path.join(scripts.BIN_DIR, 'moses-mmt-dev_2c29__c231')

    def __init__(self, ini_file):
        self._stack_size = None  # Injected
        self._cube_pruning_pop_limit = None  # Injected
        self._distortion_limit = None  # Injected

        self._moses_bin = os.path.join(Moses.bin_path, 'bin', 'moses')
        self._ini_file = ini_file

        self._server_process = None
        self._server_port = None
        self._server_proxy = None
        self._server_log_file = None

        self._features = []

    def add_feature(self, feature, name=None):
        self._features.append((feature, name))

    def __get_iniline(self, feature, name):
        custom = feature.get_iniline()
        line = feature.classname

        if name is not None:
            line += ' name=' + name

        if custom is not None:
            line += ' ' + custom

        return line

    def create_ini(self):
        lines = ['[input-factors]', '0', '', '[search-algorithm]', '1', '', '[stack]', str(self._stack_size), '',
                 '[cube-pruning-pop-limit]', str(self._cube_pruning_pop_limit), '', '[mapping]', '0 T 0', '',
                 '[distortion-limit]', str(self._distortion_limit), '', '[threads]', '${DECODER_THREADS}', '',
                 '[verbose]', '0', '', '[feature]']

        for feature in self._features:
            lines.append(self.__get_iniline(*feature))
        lines.append('')

        with open(self._ini_file, 'wb') as out:
            out.write('\n'.join(lines))
<commit_msg>Reduced Stack Size and Pop Limit to 1000<commit_after>import os

import scripts

__author__ = '<NAME>'


class MosesFeature:
    def __init__(self, classname):
        self.classname = classname

    def get_relpath(self, path):
        path = os.path.abspath(path)
        root = os.path.abspath(scripts.ENGINES_DIR)
        path = path.replace(root, '').lstrip(os.sep)

        path = os.sep.join(path.split(os.sep)[1:])

        return '${ENGINE_PATH}' + path

    def get_iniline(self):
        return None


class LexicalReordering(MosesFeature):
    def __init__(self):
        MosesFeature.__init__(self, 'LexicalReordering')

    def get_iniline(self):
        return 'input-factor=0 output-factor=0 type=hier-mslr-bidirectional-fe-allff'


class Moses:
    injector_section = 'moses'
    injectable_fields = {
        'stack_size': ('search algorithm stack size', int, 1000),
        'cube_pruning_pop_limit': ('pop limit of cube pruning algorithm', int, 1000),
        'distortion_limit': ('distortion limit', int, 6),
    }

    bin_path = os.path.join(scripts.BIN_DIR, 'moses-mmt-dev_2c29__c231')

    def __init__(self, ini_file):
        self._stack_size = None  # Injected
        self._cube_pruning_pop_limit = None  # Injected
        self._distortion_limit = None  # Injected

        self._moses_bin = os.path.join(Moses.bin_path, 'bin', 'moses')
        self._ini_file = ini_file

        self._server_process = None
        self._server_port = None
        self._server_proxy = None
        self._server_log_file = None

        self._features = []

    def add_feature(self, feature, name=None):
        self._features.append((feature, name))

    def __get_iniline(self, feature, name):
        custom = feature.get_iniline()
        line = feature.classname

        if name is not None:
            line += ' name=' + name

        if custom is not None:
            line += ' ' + custom

        return line

    def create_ini(self):
        lines = ['[input-factors]', '0', '', '[search-algorithm]', '1', '', '[stack]', str(self._stack_size), '',
                 '[cube-pruning-pop-limit]', str(self._cube_pruning_pop_limit), '', '[mapping]', '0 T 0', '',
                 '[distortion-limit]', str(self._distortion_limit), '', '[threads]', '${DECODER_THREADS}', '',
                 '[verbose]', '0', '', '[feature]']

        for feature in self._features:
            lines.append(self.__get_iniline(*feature))
        lines.append('')

        with open(self._ini_file, 'wb') as out:
            out.write('\n'.join(lines))

{'fraction_non_alphanumeric': 0.13878513858855906, 'fraction_numerical': 0.00982897582071948, 'mean_word_length': 2.593220338983051, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1833046', 'n_tokens_mistral': 1700, 'n_tokens_neox': 1606, 'n_words': 337}