<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import re
from datetime import datetime

from dateutil import tz

from arrow import locales

try:
    from functools import lru_cache
except ImportError:  # pragma: no cover
    from backports.functools_lru_cache import lru_cache  # pragma: no cover


class ParserError(ValueError):
    pass


# Allows for ParserErrors to be propagated from _build_datetime()
# when day_of_year errors occur.
# Before this, the ParserErrors were caught by the try/except in
# _parse_multiformat() and the appropriate error message was not
# transmitted to the user.
class ParserMatchError(ParserError):
    pass


class DateTimeParser(object):

    _FORMAT_RE = re.compile(
        r"(YYY?Y?|MM?M?M?|Do|DD?D?D?|d?d?d?d|HH?|hh?|mm?|ss?|S+|ZZ?Z?|a|A|X)"
    )
    _ESCAPE_RE = re.compile(r"\[[^\[\]]*\]")

    _ONE_OR_TWO_DIGIT_RE = re.compile(r"\d{1,2}")
    _ONE_OR_TWO_OR_THREE_DIGIT_RE = re.compile(r"\d{1,3}")
    _ONE_OR_MORE_DIGIT_RE = re.compile(r"\d+")
    _TWO_DIGIT_RE = re.compile(r"\d{2}")
    _THREE_DIGIT_RE = re.compile(r"\d{3}")
    _FOUR_DIGIT_RE = re.compile(r"\d{4}")
    _TZ_Z_RE = re.compile(r"([\+\-])(\d{2})(?:(\d{2}))?|Z")
    _TZ_ZZ_RE = re.compile(r"([\+\-])(\d{2})(?:\:(\d{2}))?|Z")
    _TZ_NAME_RE = re.compile(r"\w[\w+\-/]+")
    # NOTE: timestamps cannot be parsed from natural language strings (by removing the ^...$) because it will
    # break cases like "15 Jul 2000" and a format list (see issue #447)
    _TIMESTAMP_RE = re.compile(r"^\d+\.?\d+$")
    _TIME_RE = re.compile(r"^(\d{2})(?:\:?(\d{2}))?(?:\:?(\d{2}))?(?:([\.\,])(\d+))?$")

    _BASE_INPUT_RE_MAP = {
        "YYYY": _FOUR_DIGIT_RE,
        "YY": _TWO_DIGIT_RE,
        "MM": _TWO_DIGIT_RE,
        "M": _ONE_OR_TWO_DIGIT_RE,
        "DDDD": _THREE_DIGIT_RE,
        "DDD": _ONE_OR_TWO_OR_THREE_DIGIT_RE,
        "DD": _TWO_DIGIT_RE,
        "D": _ONE_OR_TWO_DIGIT_RE,
        "HH": _TWO_DIGIT_RE,
        "H": _ONE_OR_TWO_DIGIT_RE,
        "hh": _TWO_DIGIT_RE,
        "h": _ONE_OR_TWO_DIGIT_RE,
        "mm": _TWO_DIGIT_RE,
        "m": _ONE_OR_TWO_DIGIT_RE,
        "ss": _TWO_DIGIT_RE,
        "s": _ONE_OR_TWO_DIGIT_RE,
        "X": _TIMESTAMP_RE,
        "ZZZ": _TZ_NAME_RE,
        "ZZ": _TZ_ZZ_RE,
        "Z": _TZ_Z_RE,
        "S": _ONE_OR_MORE_DIGIT_RE,
    }

    SEPARATORS = ["-", "/", "."]

    def __init__(self, locale="en_us", cache_size=0):

        self.locale = locales.get_locale(locale)
        self._input_re_map = self._BASE_INPUT_RE_MAP.copy()
        self._input_re_map.update(
            {
                "MMMM": self._choice_re(self.locale.month_names[1:], re.IGNORECASE),
                "MMM": self._choice_re(
                    self.locale.month_abbreviations[1:], re.IGNORECASE
                ),
                "Do": re.compile(self.locale.ordinal_day_re),
                "dddd": self._choice_re(self.locale.day_names[1:], re.IGNORECASE),
                "ddd": self._choice_re(
                    self.locale.day_abbreviations[1:], re.IGNORECASE
                ),
                "d": re.compile(r"[1-7]"),
                "a": self._choice_re(
                    (self.locale.meridians["am"], self.locale.meridians["pm"])
                ),
                # note: 'A' token accepts both 'am/pm' and 'AM/PM' formats to
                # ensure backwards compatibility of this token
                "A": self._choice_re(self.locale.meridians.values()),
            }
        )
        if cache_size > 0:
            self._generate_pattern_re = lru_cache(maxsize=cache_size)(
                self._generate_pattern_re
            )

    # TODO: since we support more than ISO-8601, we should rename this function
    # IDEA: break into multiple functions
    def parse_iso(self, datetime_string):
        # TODO: add a flag to normalize whitespace (useful in logs, ref issue #421)
        has_space_divider = " " in datetime_string
        has_t_divider = "T" in datetime_string

        num_spaces = datetime_string.count(" ")
        if (has_space_divider and num_spaces != 1) or (
            has_t_divider and num_spaces > 0
        ):
            raise ParserError(
                "Expected an ISO 8601-like string, but was given '{}'. Try passing in a format string to resolve this.".format(
                    datetime_string
                )
            )
<commit_msg>Consolidate and clean up errors for spacing<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import re
from datetime import datetime

from dateutil import tz

from arrow import locales

try:
    from functools import lru_cache
except ImportError:  # pragma: no cover
    from backports.functools_lru_cache import lru_cache  # pragma: no cover


class ParserError(ValueError):
    pass


# Allows for ParserErrors to be propagated from _build_datetime()
# when day_of_year errors occur.
# Before this, the ParserErrors were caught by the try/except in
# _parse_multiformat() and the appropriate error message was not
# transmitted to the user.
class ParserMatchError(ParserError):
    pass


class DateTimeParser(object):

    _FORMAT_RE = re.compile(
        r"(YYY?Y?|MM?M?M?|Do|DD?D?D?|d?d?d?d|HH?|hh?|mm?|ss?|S+|ZZ?Z?|a|A|X)"
    )
    _ESCAPE_RE = re.compile(r"\[[^\[\]]*\]")

    _ONE_OR_TWO_DIGIT_RE = re.compile(r"\d{1,2}")
    _ONE_OR_TWO_OR_THREE_DIGIT_RE = re.compile(r"\d{1,3}")
    _ONE_OR_MORE_DIGIT_RE = re.compile(r"\d+")
    _TWO_DIGIT_RE = re.compile(r"\d{2}")
    _THREE_DIGIT_RE = re.compile(r"\d{3}")
    _FOUR_DIGIT_RE = re.compile(r"\d{4}")
    _TZ_Z_RE = re.compile(r"([\+\-])(\d{2})(?:(\d{2}))?|Z")
    _TZ_ZZ_RE = re.compile(r"([\+\-])(\d{2})(?:\:(\d{2}))?|Z")
    _TZ_NAME_RE = re.compile(r"\w[\w+\-/]+")
    # NOTE: timestamps cannot be parsed from natural language strings (by removing the ^...$) because it will
    # break cases like "15 Jul 2000" and a format list (see issue #447)
    _TIMESTAMP_RE = re.compile(r"^\d+\.?\d+$")
    _TIME_RE = re.compile(r"^(\d{2})(?:\:?(\d{2}))?(?:\:?(\d{2}))?(?:([\.\,])(\d+))?$")

    _BASE_INPUT_RE_MAP = {
        "YYYY": _FOUR_DIGIT_RE,
        "YY": _TWO_DIGIT_RE,
        "MM": _TWO_DIGIT_RE,
        "M": _ONE_OR_TWO_DIGIT_RE,
        "DDDD": _THREE_DIGIT_RE,
        "DDD": _ONE_OR_TWO_OR_THREE_DIGIT_RE,
        "DD": _TWO_DIGIT_RE,
        "D": _ONE_OR_TWO_DIGIT_RE,
        "HH": _TWO_DIGIT_RE,
        "H": _ONE_OR_TWO_DIGIT_RE,
        "hh": _TWO_DIGIT_RE,
        "h": _ONE_OR_TWO_DIGIT_RE,
        "mm": _TWO_DIGIT_RE,
        "m": _ONE_OR_TWO_DIGIT_RE,
        "ss": _TWO_DIGIT_RE,
        "s": _ONE_OR_TWO_DIGIT_RE,
        "X": _TIMESTAMP_RE,
        "ZZZ": _TZ_NAME_RE,
        "ZZ": _TZ_ZZ_RE,
        "Z": _TZ_Z_RE,
        "S": _ONE_OR_MORE_DIGIT_RE,
    }

    SEPARATORS = ["-", "/", "."]

    def __init__(self, locale="en_us", cache_size=0):

        self.locale = locales.get_locale(locale)
        self._input_re_map = self._BASE_INPUT_RE_MAP.copy()
        self._input_re_map.update(
            {
                "MMMM": self._choice_re(self.locale.month_names[1:], re.IGNORECASE),
                "MMM": self._choice_re(
                    self.locale.month_abbreviations[1:], re.IGNORECASE
                ),
                "Do": re.compile(self.locale.ordinal_day_re),
                "dddd": self._choice_re(self.locale.day_names[1:], re.IGNORECASE),
                "ddd": self._choice_re(
                    self.locale.day_abbreviations[1:], re.IGNORECASE
                ),
                "d": re.compile(r"[1-7]"),
                "a": self._choice_re(
                    (self.locale.meridians["am"], self.locale.meridians["pm"])
                ),
                # note: 'A' token accepts both 'am/pm' and 'AM/PM' formats to
                # ensure backwards compatibility of this token
                "A": self._choice_re(self.locale.meridians.values()),
            }
        )
        if cache_size > 0:
            self._generate_pattern_re = lru_cache(maxsize=cache_size)(
                self._generate_pattern_re
            )

    # TODO: since we support more than ISO-8601, we should rename this function
    # IDEA: break into multiple functions
    def parse_iso(self, datetime_string):
        # TODO: add a flag to normalize whitespace (useful in logs, ref issue #421)
        has_space_divider = " " in datetime_string
        has_t_divider = "T" in datetime_string

        num_spaces = datetime_string.count(" ")
        if has_space_divider and num_spaces != 1 or has_t_divider and num_spaces > 0:
            raise ParserError(
                "Expected an ISO 8601-like string, but was given '{}'. Try passing in a format string to resolve this.".format(
                    datetime_string
                )
            )

{'fraction_non_alphanumeric': 0.14194867136043607, 'fraction_numerical': 0.010220304337951397, 'mean_word_length': 2.4201941747572815, 'pattern_counts': {'":': 58, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1724323', 'n_tokens_mistral': 3385, 'n_tokens_neox': 3193, 'n_words': 622}