<commit_before>from nala.features import FeatureGenerator
import re


class TmVarFeatureGenerator(FeatureGenerator):
    """
    Generates tmVar CRF features based on the value of the token itself.
    * w[0] = the value of the words itself
    This FeatureGenerator mainly uses Regular Expressions (which are initialized in the constructor).
    The generate function iterates over the dataset and creates the features which all get separately created in their own function.

    Hints:
    - when a feature is not found (as in: regex don't apply) then the feature gets the value None.

    Implements the abstract class FeatureGenerator.
    """
    def __init__(self):
        """
        Contains all regular expressions.
        """
        self.reg_spec_chars = re.compile('.*[-;:,.>+_].*')
        self.reg_chr_keys = re.compile('.*(<KEY>')
        self.reg_char_simple_bracket = re.compile('.*[\(\)].*')
        self.reg_char_square_bracket = re.compile('.*[\[\]].*')
        self.reg_char_curly_bracket = re.compile('.*[\{\}].*')
        self.reg_char_slashes = re.compile('.*[\/\\\].*')
        self.reg_mutat_type = re.compile('.*(del|ins|dup|tri|qua|con|delins|indel).*')
        self.reg_frameshift_type = re.compile('.*(fs|fsX|fsx).*')
        self.reg_mutat_word = re.compile(
            '^(deletion|delta|elta|insertion|repeat|inversion|deletions|insertions|repeats|inversions).*')
        self.reg_mutat_article = re.compile(
            '^(single|a|one|two|three|four|five|six|seven|eight|nine|ten|[0-9]+|[0-9]+\.[0-9]+).*')
        self.reg_mutat_byte = re.compile('.*(kb|mb).*')
        self.reg_mutat_basepair = re.compile(
            '.*(base|bases|pair|amino|acid|acids|codon|postion|postions|bp|nucleotide|nucleotides).*')
        self.reg_type1 = re.compile('^[cgrm]$')
        self.reg_type12 = re.compile('^(ivs|ex|orf)$')
        self.reg_dna_symbols = re.compile('^[ATCGUatcgu]$')
        self.reg_prot_symbols1 = re.compile(
            '.*(glutamine|glutamic|leucine|valine|isoleucine|lysine|alanine|glycine|aspartate|methionine|threonine|histidine|aspartic|asparticacid|arginine|asparagine|tryptophan|proline|phenylalanine|cysteine|serine|glutamate|tyrosine|stop|frameshift).*')
        self.reg_prot_symbols2 = re.compile(
            '^(cys|ile|ser|gln|met|asn|pro|lys|asp|thr|phe|ala|gly|his|leu|arg|trp|val|glu|tyr|fs|fsx)$')
        self.reg_prot_symbols3 = re.compile('^(ys|le|er|ln|et|sn|ro|ys|sp|hr|he|la|ly|is|eu|rg|rp|al|lu|yr)$')
        self.reg_prot_symbols4 = re.compile('^[CISQMNPKDTFAGHLRWVEYX]$')
        self.reg_rs_code1 = re.compile('^(rs|RS|Rs)[0-9].*')
        self.reg_rs_code2 = re.compile('^(rs|RS|Rs)$')

        # patterns
        self.reg_shape_uc = re.compile('[A-Z]')
        self.reg_shape_uc_plus = re.compile('[A-Z]+')

        self.reg_shape_lc = re.compile('[a-z]')
        self.reg_shape_lc_plus = re.compile('[a-z]+')

        self.reg_shape_nr = re.compile('[0-9]')
        self.reg_shape_nr_plus = re.compile('[0-9]+')

        self.reg_shape_chars = re.compile('[A-Za-z]')
        self.reg_shape_chars_plus = re.compile('[A-Za-z]+')

    def generate(self, dataset):
        """
        :type dataset: structures.data.Dataset
        """
        last_token_str = ""
        for token in dataset.tokens():

            # nr of digits
            token.features['num_nr[0]'] = self.n_nr_chars(token.word)

            # nr of upper case
            token.features['num_up[0]'] = self.n_upper_chars(token.word)

            # nr of lower case
            token.features['num_lo[0]'] = self.n_lower_chars(token.word)

            # nr of letters
            token.features['num_alpha[0]'] = self.n_chars(token.word)

            # nr of specific chars: ";:,.->+_"
            token.features['num_spec_chars[0]'] = self.spec_chars(token.word)

            # chromosomal keytokens
            token.features['num_has_chr_key[0]'] = self.is_chr_key(token.word)

            # mutation type
            token.features['mutat_type[0]'] = self.mutation_type(token.word)

            # mutation word
            token.features['mutat_word[0]'] = self.mutation_word(token.word)

            # mutation article and basepair
            token.features['mutat_article_bp[0]'] = self.mutation_article_bp(token.word)

            # type 1
            token.features['type1[0]'] = self.type1(token.word)

            # type 2
            token.features['type2[0]'] = self.type2(token.word)

            # dna symbols
            token.features['dna_symbols[0]'] = self.dna_symbols(token.word)

            # protein symbols
            token.features['protein_symbols[0]'] = self.protein_symbols(token.word, last_token_str)

            # RScode
            token.features['rs_code[0]'] = self.rscode(token.word)

            # patterns
            # TODO patterns
            token.features['shape1[0]'] = self.rscode(token.word)

            # prefix patterns
            # TODO prefix patterns

            # suffix patterns
            # TODO suffix patterns

            # last token
            last_token_str = token.word

    def n_lower_chars(self, str):<commit_msg>[#18] Finish implementation of prefix suffix patterns<commit_after>from nala.features import FeatureGenerator
import re


class TmVarFeatureGenerator(FeatureGenerator):
    """
    Generates tmVar CRF features based on the value of the token itself.
    * w[0] = the value of the words itself
    This FeatureGenerator mainly uses Regular Expressions (which are initialized in the constructor).
    The generate function iterates over the dataset and creates the features which all get separately created in their own function.

    Hints:
    - when a feature is not found (as in: regex don't apply) then the feature gets the value None.

    Implements the abstract class FeatureGenerator.
    """
    def __init__(self):
        """
        Contains all regular expressions.
        """
        self.reg_spec_chars = re.compile('.*[-;:,.>+_].*')
        self.reg_chr_keys = re.compile('.*(q|p|q[0-9]+|p[0-9]+|qter|pter|XY|t).*')
        self.reg_char_simple_bracket = re.compile('.*[\(\)].*')
        self.reg_char_square_bracket = re.compile('.*[\[\]].*')
        self.reg_char_curly_bracket = re.compile('.*[\{\}].*')
        self.reg_char_slashes = re.compile('.*[\/\\\].*')
        self.reg_mutat_type = re.compile('.*(del|ins|dup|tri|qua|con|delins|indel).*')
        self.reg_frameshift_type = re.compile('.*(fs|fsX|fsx).*')
        self.reg_mutat_word = re.compile(
            '^(deletion|delta|elta|insertion|repeat|inversion|deletions|insertions|repeats|inversions).*')
        self.reg_mutat_article = re.compile(
            '^(single|a|one|two|three|four|five|six|seven|eight|nine|ten|[0-9]+|[0-9]+\.[0-9]+).*')
        self.reg_mutat_byte = re.compile('.*(kb|mb).*')
        self.reg_mutat_basepair = re.compile(
            '.*(base|bases|pair|amino|acid|acids|codon|postion|postions|bp|nucleotide|nucleotides).*')
        self.reg_type1 = re.compile('^[cgrm]$')
        self.reg_type12 = re.compile('^(ivs|ex|orf)$')
        self.reg_dna_symbols = re.compile('^[ATCGUatcgu]$')
        self.reg_prot_symbols1 = re.compile(
            '.*(glutamine|glutamic|leucine|valine|isoleucine|lysine|alanine|glycine|aspartate|methionine|threonine|histidine|aspartic|asparticacid|arginine|asparagine|tryptophan|proline|phenylalanine|cysteine|serine|glutamate|tyrosine|stop|frameshift).*')
        self.reg_prot_symbols2 = re.compile(
            '^(cys|ile|ser|gln|met|asn|pro|lys|asp|thr|phe|ala|gly|his|leu|arg|trp|val|glu|tyr|fs|fsx)$')
        self.reg_prot_symbols3 = re.compile('^(ys|le|er|ln|et|sn|ro|ys|sp|hr|he|la|ly|is|eu|rg|rp|al|lu|yr)$')
        self.reg_prot_symbols4 = re.compile('^[CISQMNPKDTFAGHLRWVEYX]$')
        self.reg_rs_code1 = re.compile('^(rs|RS|Rs)[0-9].*')
        self.reg_rs_code2 = re.compile('^(rs|RS|Rs)$')

        # patterns
        self.reg_shape_uc = re.compile('[A-Z]')
        self.reg_shape_uc_plus = re.compile('[A-Z]+')

        self.reg_shape_lc = re.compile('[a-z]')
        self.reg_shape_lc_plus = re.compile('[a-z]+')

        self.reg_shape_nr = re.compile('[0-9]')
        self.reg_shape_nr_plus = re.compile('[0-9]+')

        self.reg_shape_chars = re.compile('[A-Za-z]')
        self.reg_shape_chars_plus = re.compile('[A-Za-z]+')

    def generate(self, dataset):
        """
        :type dataset: structures.data.Dataset
        """
        last_token_str = ""
        for token in dataset.tokens():

            # nr of digits
            token.features['num_nr[0]'] = self.n_nr_chars(token.word)

            # nr of upper case
            token.features['num_up[0]'] = self.n_upper_chars(token.word)

            # nr of lower case
            token.features['num_lo[0]'] = self.n_lower_chars(token.word)

            # nr of letters
            token.features['num_alpha[0]'] = self.n_chars(token.word)

            # nr of specific chars: ";:,.->+_"
            token.features['num_spec_chars[0]'] = self.spec_chars(token.word)

            # chromosomal keytokens
            token.features['num_has_chr_key[0]'] = self.is_chr_key(token.word)

            # mutation type
            token.features['mutat_type[0]'] = self.mutation_type(token.word)

            # mutation word
            token.features['mutat_word[0]'] = self.mutation_word(token.word)

            # mutation article and basepair
            token.features['mutat_article_bp[0]'] = self.mutation_article_bp(token.word)

            # type 1
            token.features['type1[0]'] = self.type1(token.word)

            # type 2
            token.features['type2[0]'] = self.type2(token.word)

            # dna symbols
            token.features['dna_symbols[0]'] = self.dna_symbols(token.word)

            # protein symbols
            token.features['protein_symbols[0]'] = self.protein_symbols(token.word, last_token_str)

            # RScode
            token.features['rs_code[0]'] = self.rscode(token.word)

            # patterns
            # TODO patterns
            token.features['shape1[0]'] = self.rscode(token.word)

            # prefix patterns
            for index, value in enumerate(self.prefix_pattern(token.word)):
                token.features['prefix{}[0]'.format(index+1)] = value

            # suffix patterns
            for index, value in enumerate(self.suffix_pattern(token.word)):
                token.features['suffix{}[0]'.format(index+1)] = value

            # last token
            last_token_str = token.word

    def n_lower_chars(self, str):
{'fraction_non_alphanumeric': 0.1526146267525578, 'fraction_numerical': 0.009283819628647215, 'mean_word_length': 2.7356687898089174, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2935132', 'n_tokens_mistral': 3734, 'n_tokens_neox': 3521, 'n_words': 517}