<commit_before>#!/usr/bin/env python3

from collections import OrderedDict
from enum import Enum
from itertools import chain
from typing import Iterable, List, Mapping, Sequence, Tuple


class Label(Enum):
    O = 0
    B = 1
    I = 2


def label_from_string(s: str) -> Label:
    for label in Label:
        if s.upper().replace('-', '_') == label.name:
            return label
    else:
        raise ValueError(f"invalid label: {s}")


class DataToken:
    def __init__(self, offset, word, lowercase, pos_tag, mwe_tag, parent_offset, sentence_id=''):
        self.offset = int(offset)
        self.word = word
        self.lowercase_lemma = lowercase
        self.pos_tag = pos_tag
        self.mwe_tag = label_from_string(mwe_tag)
        self.parent_offset = 0 if not parent_offset else int(parent_offset)
        self.sentence_id = sentence_id

    def __repr__(self):
        return '\t'.join(map(lambda w: str(w) if w is not None else '', [
            self.offset, self.word, self.lowercase_lemma, self.pos_tag, self.mwe_tag, self.parent_offset,
            self.sentence_id
        ]))

    def __str__(self):
        return repr(self)

def make_phi(sentence_id='') -> DataToken:
    return DataToken(0, 'PHI', 'PHI', 'PHIPOS', 'O', '', sentence_id)


def make_omega(sentence_id='') -> DataToken:
    return DataToken(0, 'OMEGA', 'OMEGA', 'OMEGAPOS', 'O', '', sentence_id)


def make_unknown(sentence_id='') -> DataToken:
    return DataToken(0, 'UNKWORD', 'UNKWORD', 'UNKPOS', 'O', '', sentence_id)


PHI = make_phi()
OMEGA = make_omega()
UNKNOWN = make_unknown()


class DataSentence(Sequence):
    def __init__(self):
        self._tokens: List[DataToken] = list()
        self.sentence_id = None

    def __iter__(self) -> Iterable[DataToken]:
        return iter(self._tokens)

    def __getitem__(self, item) -> DataToken:
        if item < 0:
            return make_phi(self.sentence_id)
        elif item >= len(self._tokens):
            return make_omega(self.sentence_id)
        else:
            return self._tokens[item]

    def __bool__(self) -> bool:
        return bool(self._tokens)

    def __len__(self) -> int:
        return len(self._tokens)

    def append(self, token: DataToken):
        if not self.sentence_id:
            self.sentence_id = token.sentence_id
        else:
            if token.sentence_id != self.sentence_id:
                raise ValueError(f"sentence IDs do not match: {self.sentence_id} (sent) != {token.sentence_id} (token)")
        self._tokens.append(token)

    @property
    def sentence(self) -> str:
        return ' '.join(map(lambda t: t.word, self._tokens))


class FeatureSet(Mapping):
    def __init__(self):
        self._features = {}

    def __len__(self):
        return len(self._features)

    def __iter__(self):
        return iter(self._features)

    def __getitem__(self, feature):
        if feature not in self._features:
            raise KeyError(f"no such feature in feature set: {feature}")
        return self._features[feature]

    def add(self, feature):
        if feature in self._features:
            return self._features[feature]
        self._features[feature] = len(self) + 1
        return len(self)

    def get(self, key, default=None):
        return self._features.get(key, default)


class FeatureVector:
    def __init__(self, label: Label):
        self.label = label
        self._features = set()

    def add(self, feature: int):
        self._features.add(feature)

    def __str__(self):
        return f'{self.label.value} {" ".join((feature for feature in sorted(self._features)))}'


class Classifier:
    def __init__(self, feature_words: Iterable[str], feature_poses: Iterable[str]):
        self.feature_set = FeatureSet()
        self._generate_features(feature_words, feature_poses)
        self.reset_vectors()

    def reset_vectors(self):
        self.training_vectors = list()
        self.testing_vectors = list()

    @staticmethod
    def _current_word_feature(word):<commit_msg>Fix string formatting<commit_after>#!/usr/bin/env python3

from collections import OrderedDict
from enum import Enum
from itertools import chain
from typing import Iterable, List, Mapping, Sequence, Tuple


class Label(Enum):
    O = 0
    B = 1
    I = 2


def label_from_string(s: str) -> Label:
    for label in Label:
        if s.upper().replace('-', '_') == label.name:
            return label
    else:
        raise ValueError(f"invalid label: {s}")


class DataToken:
    def __init__(self, offset, word, lowercase, pos_tag, mwe_tag, parent_offset, sentence_id=''):
        self.offset = int(offset)
        self.word = word
        self.lowercase_lemma = lowercase
        self.pos_tag = pos_tag
        self.mwe_tag = label_from_string(mwe_tag)
        self.parent_offset = 0 if not parent_offset else int(parent_offset)
        self.sentence_id = sentence_id

    def __repr__(self):
        return '\t'.join(map(lambda w: str(w) if w is not None else '', [
            self.offset, self.word, self.lowercase_lemma, self.pos_tag, self.mwe_tag, self.parent_offset,
            self.sentence_id
        ]))

    def __str__(self):
        return repr(self)

def make_phi(sentence_id='') -> DataToken:
    return DataToken(0, 'PHI', 'PHI', 'PHIPOS', 'O', '', sentence_id)


def make_omega(sentence_id='') -> DataToken:
    return DataToken(0, 'OMEGA', 'OMEGA', 'OMEGAPOS', 'O', '', sentence_id)


def make_unknown(sentence_id='') -> DataToken:
    return DataToken(0, 'UNKWORD', 'UNKWORD', 'UNKPOS', 'O', '', sentence_id)


PHI = make_phi()
OMEGA = make_omega()
UNKNOWN = make_unknown()


class DataSentence(Sequence):
    def __init__(self):
        self._tokens: List[DataToken] = list()
        self.sentence_id = None

    def __iter__(self) -> Iterable[DataToken]:
        return iter(self._tokens)

    def __getitem__(self, item) -> DataToken:
        if item < 0:
            return make_phi(self.sentence_id)
        elif item >= len(self._tokens):
            return make_omega(self.sentence_id)
        else:
            return self._tokens[item]

    def __bool__(self) -> bool:
        return bool(self._tokens)

    def __len__(self) -> int:
        return len(self._tokens)

    def append(self, token: DataToken):
        if not self.sentence_id:
            self.sentence_id = token.sentence_id
        else:
            if token.sentence_id != self.sentence_id:
                raise ValueError(f"sentence IDs do not match: {self.sentence_id} (sent) != {token.sentence_id} (token)")
        self._tokens.append(token)

    @property
    def sentence(self) -> str:
        return ' '.join(map(lambda t: t.word, self._tokens))


class FeatureSet(Mapping):
    def __init__(self):
        self._features = {}

    def __len__(self):
        return len(self._features)

    def __iter__(self):
        return iter(self._features)

    def __getitem__(self, feature):
        if feature not in self._features:
            raise KeyError(f"no such feature in feature set: {feature}")
        return self._features[feature]

    def add(self, feature):
        if feature in self._features:
            return self._features[feature]
        self._features[feature] = len(self) + 1
        return len(self)

    def get(self, key, default=None):
        return self._features.get(key, default)


class FeatureVector:
    def __init__(self, label: Label):
        self.label = label
        self._features = set()

    def add(self, feature: int):
        self._features.add(feature)

    def __str__(self):
        return f'{self.label.value} {" ".join((str(feature) for feature in sorted(self._features)))}'


class Classifier:
    def __init__(self, feature_words: Iterable[str], feature_poses: Iterable[str]):
        self.feature_set = FeatureSet()
        self._generate_features(feature_words, feature_poses)
        self.reset_vectors()

    def reset_vectors(self):
        self.training_vectors = list()
        self.testing_vectors = list()

    @staticmethod
    def _current_word_feature(word):
{'fraction_non_alphanumeric': 0.11789578410645442, 'fraction_numerical': 0.002487252829250093, 'mean_word_length': 2.749184149184149, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241218', 'n_tokens_mistral': 2559, 'n_tokens_neox': 2447, 'n_words': 619}