<commit_before>from __future__ import division
from marmot.features.feature_extractor import FeatureExtractor


class AlphaNumericFeatureExtractor(FeatureExtractor):
    '''
    - percentage of numbers in the source
    - percentage of numbers in the target
    - absolute difference between number of numbers in the source and target sentence normalised by source sentence length
    - percentage of source words that contain non-alphabetic symbols
    - percentage of target words that contain non-alphabetic symbols
    - ratio of percentage of tokens a-z in the source and tokens a-z in the target
    '''

    def get_features(self, context_obj):
        tg_numbers = 0
        tg_alphanumeric = 0
        for word in context_obj['token']:
            if word.isalnum():
                tg_alphanumeric += 1
            try:
                float(word)
                tg_numbers += 1
            except:
                pass
        src_numbers = 0
        src_alphanumeric = 0
        if 'source_token' in context_obj and len(context_obj['source_token']) > 0:
            for word in context_obj['source_token']:
                if word.isalnum():
                    tg_alphanumeric += 1
                try:
                    float(word)
                    tg_numbers += 1
                except:
                    pass

        src_len = len(context_obj['source_token'])
        tg_len = len(context_obj['token'])
        src_num_percent = 0
        src_tg_num_diff = 0
        src_alnum_percent = 0
        src_tg_alnum_diff = 0
        if src_len > 0:
            src_num_percent = src_numbers/src_len
            src_tg_num_diff = (src_numbers - tg_numbers)/src_len
            src_alnum_percent = src_alphanumeric/src_len
            src_tg_alnum_diff = (src_alphanumeric - tg_alphanumeric)/src_len
 
        return [src_num_percent,
                tg_numbers/tg_len,
                src_tg_num_diff,
                src_alnum_percent,
                tg_alphanumeric/tg_len,
                src_tg_alnum_diff]

    def get_feature_names(self):
        return ['percentage_src_numbers',
                'percentage_tg_numbers',
                'src_tg_numbers_normalized_ratio',
                'percentage_src_alphanumeric',
                'percentage_tg_alphanumeric',
                'src_tg_alphanumeric_normalized_ratio']
<commit_msg>fix computation of alphanumeric<commit_after>from __future__ import division
from marmot.features.feature_extractor import FeatureExtractor


class AlphaNumericFeatureExtractor(FeatureExtractor):
    '''
    - percentage of numbers in the source
    - percentage of numbers in the target
    - absolute difference between number of numbers in the source and target sentence normalised by source sentence length
    - percentage of source words that contain non-alphabetic symbols
    - percentage of target words that contain non-alphabetic symbols
    - ratio of percentage of tokens a-z in the source and tokens a-z in the target
    '''

    def get_features(self, context_obj):
        tg_numbers = 0
        tg_alphanumeric = 0
        for word in context_obj['token']:
            try:
                float(word)
                tg_numbers += 1
            except:
                if word.isalnum() and not word.isalpha():
                    tg_alphanumeric += 1
        src_numbers = 0
        src_alphanumeric = 0
        if 'source_token' in context_obj and len(context_obj['source_token']) > 0:
            for word in context_obj['source_token']:
                try:
                    float(word)
                    src_numbers += 1
                except:
                    if word.isalnum() and not word.isalpha():
                        src_alphanumeric += 1
 
        src_len = len(context_obj['source_token'])
        tg_len = len(context_obj['token'])
        src_num_percent = 0
        src_tg_num_diff = 0
        src_alnum_percent = 0
        src_tg_alnum_diff = 0
        if src_len > 0:
            src_num_percent = src_numbers/src_len
            src_tg_num_diff = abs(src_numbers - tg_numbers)/src_len
            src_alnum_percent = src_alphanumeric/src_len
            src_tg_alnum_diff = abs(src_alphanumeric - tg_alphanumeric)/src_len
 
        return [src_num_percent,
                tg_numbers/tg_len,
                src_tg_num_diff,
                src_alnum_percent,
                tg_alphanumeric/tg_len,
                src_tg_alnum_diff]

    def get_feature_names(self):
        return ['percentage_src_numbers',
                'percentage_tg_numbers',
                'src_tg_numbers_normalized_ratio',
                'percentage_src_alphanumeric',
                'percentage_tg_alphanumeric',
                'src_tg_alphanumeric_normalized_ratio']

{'fraction_non_alphanumeric': 0.06046609279865631, 'fraction_numerical': 0.005878647910980474, 'mean_word_length': 1.9066503965832824, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4551891', 'n_tokens_mistral': 1408, 'n_tokens_neox': 1310, 'n_words': 321}