<commit_before># -*- coding: utf-8 -*-

import logging
import numpy as np
import re
from collections import Counter

import config
from word_dictionary import WordDictionary as WD

class Caps(object):
    """Dummy class for storing numeric values for capitalization."""
    num_values = 4
    lower = 0
    title = 1
    non_alpha = 2
    other = 3


class Token(object):
    def __init__(self, word, lemma='NA', pos='NA', morph='NA', chunk='NA'):
        """
        A token representation that stores discrete attributes to be given as 
        input to the neural network. 
        """
        self.word = word
        self.lemma = lemma
        self.pos = pos
        self.morph = morph
        self.chunk = chunk
    
    def __str__(self):
        return str(self.word)
    
    def __repr__(self):
        return self.word.__repr__()


class Suffix(object):
    """Dummy class for manipulating suffixes and their related codes."""
    codes = {}
    other = 0
    
    # initially, there is only the "other" (rare) suffix
    num_suffixes = 1
    
    @classmethod
    def load_suffixes(cls):
        """
        loads the listed suffixes from the suffix file.
        """
        Suffix.codes = {}
        code = Suffix.other + 1
        logger = logging.getLogger("Logger")
        try:
            with open(config.FILES['suffixes'], 'rb') as f:
                for line in f:
                    suffix = unicode(line.strip(), 'utf-8')
                    Suffix.codes[suffix] = code
                    code += 1
        except IOError:
            logger.warning('Suffix list doesn\'t exist.')
            raise
        
        Suffix.num_suffixes = code
    
    
    @classmethod
    def get_suffix(cls, word):
        """
        Returns the suffix code for the given word.
        
        This implementation is not the most efficient (it checks every 
        suffix separately, instead of using a tree structure), but 
        it is intended to be called only once per word.
        """
        for suffix in Suffix.codes:
            if word.endswith(suffix) and len(word) > suffix:
                return Suffix.codes[suffix]
        
        return Suffix.other
<commit_msg>correction in suffix checking<commit_after># -*- coding: utf-8 -*-

import logging
import numpy as np
import re
from collections import Counter

import config
from word_dictionary import WordDictionary as WD

class Caps(object):
    """Dummy class for storing numeric values for capitalization."""
    num_values = 4
    lower = 0
    title = 1
    non_alpha = 2
    other = 3


class Token(object):
    def __init__(self, word, lemma='NA', pos='NA', morph='NA', chunk='NA'):
        """
        A token representation that stores discrete attributes to be given as 
        input to the neural network. 
        """
        self.word = word
        self.lemma = lemma
        self.pos = pos
        self.morph = morph
        self.chunk = chunk
    
    def __str__(self):
        return str(self.word)
    
    def __repr__(self):
        return self.word.__repr__()


class Suffix(object):
    """Dummy class for manipulating suffixes and their related codes."""
    codes = {}
    suffixes_by_size = []
    other = 0
    
    # initially, there is only the "other" (rare) suffix
    num_suffixes = 1
    
    @classmethod
    def load_suffixes(cls):
        """
        loads the listed suffixes from the suffix file.
        """
        Suffix.codes = {}
        code = Suffix.other + 1
        logger = logging.getLogger("Logger")
        try:
            with open(config.FILES['suffixes'], 'rb') as f:
                for line in f:
                    suffix = unicode(line.strip(), 'utf-8')
                    Suffix.codes[suffix] = code
                    code += 1
        except IOError:
            logger.warning('Suffix list doesn\'t exist.')
            raise
        
        Suffix.num_suffixes = code
        Suffix.suffixes_by_size = sorted(Suffix.codes,
                                         key=len,
                                         reverse=True)
    
    
    @classmethod
    def get_suffix(cls, word):
        """
        Returns the suffix code for the given word.
        
        This implementation is not the most efficient (it checks every 
        suffix separately, instead of using a tree structure), but 
        it is intended to be called only once per word.
        """
        # check longer suffixes first
        for suffix in Suffix.suffixes_by_size:
            if word.endswith(suffix) and len(word) > suffix:
                return Suffix.codes[suffix]
        
        return Suffix.other

{'fraction_non_alphanumeric': 0.08672261674198407, 'fraction_numerical': 0.004734237142242307, 'mean_word_length': 1.7716159809183065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3352941', 'n_tokens_mistral': 1301, 'n_tokens_neox': 1222, 'n_words': 426}