<commit_before># -*- coding: utf-8 -*-
import re
from itertools import chain

from dateutil import parser

from dateparser.timezone_parser import pop_tz_offset_from_string
from dateparser.utils import wrap_replacement_for_regex

from .dictionary import Dictionary, ALWAYS_KEEP_TOKENS
from .validation import LanguageValidator
from ..conf import settings


class Language(object):
    _dictionary = None
    _splitters = None
    _wordchars = None
    _cached = None

    def __init__(self, shortname, language_info):
        self.shortname = shortname
        self.info = language_info.copy()
        for simplification in self.info.get('simplifications', []):
            key, value = simplification.items()[0]
            if isinstance(value, int):
                simplification[key] = str(value)

        self._cached = self

    def validate_info(self, validator=None):
        if validator is None:
            validator = LanguageValidator

        return validator.validate_info(language_id=self.shortname, info=self._cached.info)

    def is_applicable(self, date_string, strip_timezone=False):
        if strip_timezone:
            date_string, timezone = pop_tz_offset_from_string(date_string, as_offset=False)

        date_string = self._cached._simplify(date_string)
        tokens = self._cached._split(date_string, keep_formatting=False)
        if self._cached._is_date_consists_of_digits_only(tokens):
            return True
        else:
            return self._cached._are_all_words_in_the_dictionary(tokens)

    def translate(self, date_string, keep_formatting=False):
        tokens = self._cached._get_new_skip_tokens(settings.SKIP_TOKENS)

        if tokens:
            self._cached.info['skip'] += tokens
            self._cached = Language(self._language_object.shortname, self._language_object.info)

        date_string = self._cached._simplify(date_string)
        words = self._cached._split(date_string, keep_formatting)

        dictionary = self._cached._get_dictionary()<commit_msg>Moved logic to add new skip tokens to is_applicable method<commit_after># -*- coding: utf-8 -*-
import re
from itertools import chain

from dateutil import parser

from dateparser.timezone_parser import pop_tz_offset_from_string
from dateparser.utils import wrap_replacement_for_regex

from .dictionary import Dictionary, ALWAYS_KEEP_TOKENS
from .validation import LanguageValidator
from ..conf import settings


class Language(object):
    _dictionary = None
    _splitters = None
    _wordchars = None
    _cached = None

    def __init__(self, shortname, language_info):
        self.shortname = shortname
        self.info = language_info.copy()
        for simplification in self.info.get('simplifications', []):
            key, value = simplification.items()[0]
            if isinstance(value, int):
                simplification[key] = str(value)

        self._cached = self

    def validate_info(self, validator=None):
        if validator is None:
            validator = LanguageValidator

        return validator.validate_info(language_id=self.shortname, info=self._cached.info)

    def is_applicable(self, date_string, strip_timezone=False):
        tokens = self._cached._get_new_skip_tokens(settings.SKIP_TOKENS)

        if tokens:
            self._cached.info['skip'] += tokens
            self._cached = Language(self._cached.shortname, self._cached.info)

        if strip_timezone:
            date_string, timezone = pop_tz_offset_from_string(date_string, as_offset=False)

        date_string = self._cached._simplify(date_string)
        tokens = self._cached._split(date_string, keep_formatting=False)
        if self._cached._is_date_consists_of_digits_only(tokens):
            return True
        else:
            return self._cached._are_all_words_in_the_dictionary(tokens)

    def translate(self, date_string, keep_formatting=False):
        date_string = self._cached._simplify(date_string)
        words = self._cached._split(date_string, keep_formatting)

        dictionary = self._cached._get_dictionary()
{'fraction_non_alphanumeric': 0.08011796510199067, 'fraction_numerical': 0.0009830425165888424, 'mean_word_length': 3.2484342379958244, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6590793', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1130, 'n_words': 248}