<commit_before>import sys
from subprocess import Popen, PIPE

from marmot.features.feature_extractor import FeatureExtractor


class POSFeatureExtractor(FeatureExtractor):
    """
    POS for source and target words, tagged with TreeTagger
    """
    def __init__(self, tagger='', par_file_src='', par_file_tg=''):
        self.tagger = tagger
        self.par_src = par_file_src
        self.par_tg = par_file_tg

    # tag words if context_obj has no tagging
    # returns tags for all words in sentence
    def _call_tagger(self, tok_list, lang='tg'):
        par_file = self.par_tg if lang == 'tg' else self.par_src
        out = []

        if self.tagger == '' or par_file == '':
            sys.stderr.write('Tagging script and parameter file should be provided\n')
            return []

        p = Popen([self.tagger, '-quiet', par_file], stdin=PIPE, stdout=PIPE)
        out = p.communicate(input='\n'.join([tok.encode('utf-8') for tok in tok_list]))[0].decode('utf-8').split('\n')
        return out

    def get_features(self, context_obj):
        if 'target_pos' not in context_obj:
            if 'target' in context_obj and context_obj['target'] is not None:
                context_obj['target_pos'] = self._call_tagger(context_obj['target'])
            else:
                context_obj['target_pos'] = []
        if 'source_pos' not in context_obj:
            if 'source' in context_obj and context_obj['source'] is not None:
                context_obj['source_pos'] = self._call_tagger(context_obj['source'], lang='src')
            else:
                context_obj['source_pos'] = []

        # extract POS features:
        # - target POS
        # - source POS (may be more than 1)
        # - something else?
        tg_pos = context_obj['target_pos'][context_obj['index']] if context_obj['target_pos'] != [] else ''
        src_pos = []
        if 'source_pos' in context_obj and context_obj['source_pos'] != [] and 'alignments' in context_obj:
            src_pos = [context_obj['source_pos'][i] for i in context_obj['alignments'][context_obj['index']]]

        return [tg_pos, src_pos]
<commit_msg>bugfix<commit_after>import sys
from subprocess import Popen, PIPE

from marmot.features.feature_extractor import FeatureExtractor


class POSFeatureExtractor(FeatureExtractor):
    """
    POS for source and target words, tagged with TreeTagger
    """
    def __init__(self, tagger='', par_file_src='', par_file_tg=''):
        self.tagger = tagger
        self.par_src = par_file_src
        self.par_tg = par_file_tg

    # tag words if context_obj has no tagging
    # returns tags for all words in sentence
    def _call_tagger(self, tok_list, lang='tg'):
        par_file = self.par_tg if lang == 'tg' else self.par_src
        out = []

        if self.tagger == '' or par_file == '':
            sys.stderr.write('Tagging script and parameter file should be provided\n')
            return []

        p = Popen([self.tagger, '-quiet', par_file], stdin=PIPE, stdout=PIPE)
        out = p.communicate(input='\n'.join([tok.encode('utf-8') for tok in tok_list]))[0].decode('utf-8').split('\n')
        return out

    def get_features(self, context_obj):
        if 'target_pos' not in context_obj:
            if 'target' in context_obj and context_obj['target'] is not None and self.tagger is not None and self.par_tg is not None:
                context_obj['target_pos'] = self._call_tagger(context_obj['target'])
            else:
                context_obj['target_pos'] = []
        if 'source_pos' not in context_obj:
            if 'source' in context_obj and context_obj['source'] is not None and self.tagger is not None and self.par_tg is not None:
                context_obj['source_pos'] = self._call_tagger(context_obj['source'], lang='src')
            else:
                context_obj['source_pos'] = []

        # extract POS features:
        # - target POS
        # - source POS (may be more than 1)
        # - something else?
        tg_pos = context_obj['target_pos'][context_obj['index']] if context_obj['target_pos'] != [] else ''
        src_pos = []
        if 'source_pos' in context_obj and context_obj['source_pos'] != [] and 'alignments' in context_obj:
            src_pos = [context_obj['source_pos'][i] for i in context_obj['alignments'][context_obj['index']]]

        return [tg_pos, src_pos]

{'fraction_non_alphanumeric': 0.12085145342183566, 'fraction_numerical': 0.0018310826276035706, 'mean_word_length': 2.731853116994022, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2965505', 'n_tokens_mistral': 1356, 'n_tokens_neox': 1321, 'n_words': 350}