<commit_before>import abc
from nala.structures.data import Label
import re
from nala.utils import MUT_CLASS_ID


class Labeler:
    """
    Abstract class for generating labels for each token in the dataset.
    Subclasses that inherit this class should:
    * Be named [Name]Labeler
    * Implement the abstract method label
    * Append new items to the list field "original_labels" of each Token in the dataset
    """

    @abc.abstractmethod
    def label(self, dataset):
        """
        :type dataset: nala.structures.data.Dataset
        """
        return


class BIOLabeler(Labeler):
    """
    Implements a simple labeler using the annotations of the dataset
    using the BIO (beginning, inside, outside) format. Creates labels
    based on the class_id value in the Annotation object. That is:
    * B-[class_id]
    * I-[class_id]
    * O

    Requires the list field "annotations" to be previously set.
    Implements the abstract class Labeler.
    """

    def label(self, dataset):
        """
        :type dataset: nala.structures.data.Dataset
        """
        for part in dataset.parts():
            so_far = 0
            for sentence in part.sentences:
                for token in sentence:
                    token_start = part.text.find(token.word, so_far)
                    so_far = token_start + len(token.word)
                    token.original_labels = [Label('O')]

                    for ann in part.annotations:
                        start = ann.offset
                        end = ann.offset + len(ann.text)
                        if start == token_start:
                            token.original_labels[0].value = 'B-{}'.format(ann.class_id)
                            break
                        elif start < token_start < end:
                            token.original_labels[0].value = 'I-{}'.format(ann.class_id)
                            break


class TmVarLabeler(Labeler):
    """
    Implements a labeler using the annotations of the dataset
    based on the labeling scheme used by tmVar (http://www.ncbi.nlm.nih.gov/CBBresearch/Lu/Demo/tmTools/#tmVar)

    This is however a close approximation of their lableing scheme and not an exact replica.

    The following labels are possible:
    * A - Reference sequence
    * T - Mutation type
    * F - Frame shift
    * R - SNP
    * M - Mutant
    * W - Wild type
    * S - Frame shift position
    * P - Mutation position
    * I - Other inside mutation tokens
    * O - Outside

    Requires the list field "annotations" to be previously set.
    Requires the TmVarTokenizer to be used.
    Implements the abstract class Labeler.
    """

    def __init__(self):
        # A
        self.label_reference_sequence = re.compile('^([cgrmp])|(ivs|ex|orf)$')
        # T
        self.label_type = re.compile('(del|ins|dup|tri|qua|con|delins|indel)')
        # F
        self.label_frameshift = re.compile('^(fs|fsX|fsx)$')
        # R
        self.label_snip = re.compile('^(rs|RS|Rs)$')

        # W or M (wild type or mutant)
        self.dna_symbols = re.compile('^[ATCGUatcgu]+$')
        self.protein_symbols = re.compile('(glutamine|glutamic|leucine|valine|isoleucine|lysine|alanine|glycine|'
                                          'aspartate|methionine|threonine|histidine|aspartic|asparticacid|arginine|'
                                          'asparagine|tryptophan|proline|phenylalanine|cysteine|serine|glutamate|'
                                          'tyrosine|stop|frameshift)|(^(cys|ile|ser|gln|met|asn|pro|lys|asp|thr|phe|'
                                          'ala|gly|his|leu|arg|trp|val|glu|tyr|fs|fsx)$)|(^[cisqmnpkdtfaghlrwveyx]$)')

        # P or S (mutation_position or frameshift_position)
        self.position = re.compile('^[0-9]+$')

    def _match_regex_label(self, previous_token, token):
        if self.label_reference_sequence.search(token.word):
            token.original_labels[0].value = 'A'  # Reference sequence
        elif self.label_type.search(token.word):
            token.original_labels[0].value = 'T'  # Mutation type
        elif self.label_frameshift.search(token.word):
            token.original_labels[0].value = 'F'  # Frame shift
        elif previous_token is not None and previous_token.original_labels[0].value is 'F' and token.word is 'X':<commit_msg>[#60] Revert previous "improvement"<commit_after>import abc
from nala.structures.data import Label
import re
from nala.utils import MUT_CLASS_ID


class Labeler:
    """
    Abstract class for generating labels for each token in the dataset.
    Subclasses that inherit this class should:
    * Be named [Name]Labeler
    * Implement the abstract method label
    * Append new items to the list field "original_labels" of each Token in the dataset
    """

    @abc.abstractmethod
    def label(self, dataset):
        """
        :type dataset: nala.structures.data.Dataset
        """
        return


class BIOLabeler(Labeler):
    """
    Implements a simple labeler using the annotations of the dataset
    using the BIO (beginning, inside, outside) format. Creates labels
    based on the class_id value in the Annotation object. That is:
    * B-[class_id]
    * I-[class_id]
    * O

    Requires the list field "annotations" to be previously set.
    Implements the abstract class Labeler.
    """

    def label(self, dataset):
        """
        :type dataset: nala.structures.data.Dataset
        """
        for part in dataset.parts():
            so_far = 0
            for sentence in part.sentences:
                for token in sentence:
                    token_start = part.text.find(token.word, so_far)
                    so_far = token_start + len(token.word)
                    token.original_labels = [Label('O')]

                    for ann in part.annotations:
                        start = ann.offset
                        end = ann.offset + len(ann.text)
                        if start == token_start:
                            token.original_labels[0].value = 'B-{}'.format(ann.class_id)
                            break
                        elif start < token_start < end:
                            token.original_labels[0].value = 'I-{}'.format(ann.class_id)
                            break


class TmVarLabeler(Labeler):
    """
    Implements a labeler using the annotations of the dataset
    based on the labeling scheme used by tmVar (http://www.ncbi.nlm.nih.gov/CBBresearch/Lu/Demo/tmTools/#tmVar)

    This is however a close approximation of their lableing scheme and not an exact replica.

    The following labels are possible:
    * A - Reference sequence
    * T - Mutation type
    * F - Frame shift
    * R - SNP
    * M - Mutant
    * W - Wild type
    * S - Frame shift position
    * P - Mutation position
    * I - Other inside mutation tokens
    * O - Outside

    Requires the list field "annotations" to be previously set.
    Requires the TmVarTokenizer to be used.
    Implements the abstract class Labeler.
    """

    def __init__(self):
        # A
        self.label_reference_sequence = re.compile('(^[cgrm]$)|(^(ivs|ex|orf)$)')
        # T
        self.label_type = re.compile('(del|ins|dup|tri|qua|con|delins|indel)')
        # F
        self.label_frameshift = re.compile('^(fs|fsX|fsx)$')
        # R
        self.label_snip = re.compile('^(rs|RS|Rs)$')

        # W or M (wild type or mutant)
        self.dna_symbols = re.compile('^[ATCGUatcgu]+$')
        self.protein_symbols = re.compile('(glutamine|glutamic|leucine|valine|isoleucine|lysine|alanine|glycine|'
                                          'aspartate|methionine|threonine|histidine|aspartic|asparticacid|arginine|'
                                          'asparagine|tryptophan|proline|phenylalanine|cysteine|serine|glutamate|'
                                          'tyrosine|stop|frameshift)|(^(cys|ile|ser|gln|met|asn|pro|lys|asp|thr|phe|'
                                          'ala|gly|his|leu|arg|trp|val|glu|tyr|fs|fsx)$)|(^[cisqmnpkdtfaghlrwveyx]$)')

        # P or S (mutation_position or frameshift_position)
        self.position = re.compile('^[0-9]+$')

    def _match_regex_label(self, previous_token, token):
        if self.label_reference_sequence.search(token.word):
            token.original_labels[0].value = 'A'  # Reference sequence
        elif self.label_type.search(token.word):
            token.original_labels[0].value = 'T'  # Mutation type
        elif self.label_frameshift.search(token.word):
            token.original_labels[0].value = 'F'  # Frame shift
        elif previous_token is not None and previous_token.original_labels[0].value is 'F' and token.word is 'X':
{'fraction_non_alphanumeric': 0.10143595634692705, 'fraction_numerical': 0.0022975301550832855, 'mean_word_length': 2.1026372059871705, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '642741', 'n_tokens_mistral': 2646, 'n_tokens_neox': 2490, 'n_words': 646}