<commit_before>import string
import textwrap
import binascii

from veryprettytable import VeryPrettyTable

from plugins import BasePlugin


__author__ = 'peter'


class AlphabetPlugin(BasePlugin):
    short_description = 'Using the string as index in the alphabet'
    default = True
    description = textwrap.dedent('''\
    This plugin only works when (at least one of the) string(s) is either a concatenation of decimals, or a hex string.
    This plugin will try to use character pairs from the string as indexes in the alphabet, such that 01 is 'a',
    02 is 'b', etc. If the string is a hexstring the pairs will first be converted to integers.''')

    def sentinel(self):
        for s in self.args['STRING']:
            if all(x.isdigit() for x in s):
                # All digits
                if all(0 <= int(x) <= 26 for s in self.args['STRING'] for x in self._chunks(s, 2)):
                    # All between 0 and 26
                    return True
<commit_msg>Fix bug in alphabet plugin<commit_after>import string
import textwrap
import binascii

from veryprettytable import VeryPrettyTable

from plugins import BasePlugin


__author__ = 'peter'


class AlphabetPlugin(BasePlugin):
    short_description = 'As index in the alphabet:'
    default = True
    description = textwrap.dedent('''\
    This plugin only works when (at least one of the) string(s) is either a concatenation of decimals, or a hex string.
    This plugin will try to use character pairs from the string as indexes in the alphabet, such that 01 is 'a',
    02 is 'b', etc. If the string is a hexstring the pairs will first be converted to integers.''')

    def sentinel(self):
        for s in self.args['STRING']:
            if all(x.isdigit() for x in s):
                # All digits
                if all(0 <= int(x) <= 26 for x in self._chunks(s, 2)):
                    # All between 0 and 26
                    return True

{'fraction_non_alphanumeric': 0.07476635514018691, 'fraction_numerical': 0.01142263759086189, 'mean_word_length': 2.601869158878505, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4642426', 'n_tokens_mistral': 570, 'n_tokens_neox': 512, 'n_words': 247}