<commit_before>import enum
import ciso8601
import time
# noinspection PyUnresolvedReferences
import re  # noqa
from collections import Counter

# noinspection PyUnresolvedReferences
from .common import *  # noqa
from .. import pd


class DataPoint:
    """Dynamically generated datapoint class"""

    def items(self):
        """Returns an iterator over pair of keys and values"""

    def to_dict(self) -> dict:
        """Converts datapoint to a regular dictionary"""

    def to_lineprotocol(self) -> bytes:
        """Returns InfluxDB line protocol representation of datapoint"""


class InfluxType(enum.Enum):
    MEASUREMENT = 0
    TIMEINT = 10
    TIMESTR = 11
    TIMEDT = 12
    TAG = 20
    # Fields (>=30)
    BOOL = 30
    INT = 40
    DATETIME = 41
    TIMEDELTA = 42
    FLOAT = 50
    STR = 60
    ENUM = 61


def str_to_dt(s):
    dt = ciso8601.parse_datetime(s)
    if dt:
        return dt
    raise ValueError(f'Invalid datetime string: {dt!r}')


def dt_to_int(dt):
    if not dt.tzinfo:
        # Assume tz-naive input to be in UTC, not local time
        return int(dt.timestamp() - time.timezone) * 10 ** 9 + dt.microsecond * 1000
    return int(dt.timestamp()) * 10 ** 9 + dt.microsecond * 1000


def td_to_int(td):
    return int(td.total_seconds()) * 10 ** 9 + td.microseconds * 1000


def _gen_parser(schema, meas, rm_none=False, extra_tags=None):
    """Factory of datapoint -> line protocol parsers"""
    tags = []
    fields = []
    ts = None
    meas = meas
    for k, t in schema.items():
        if t is InfluxType.MEASUREMENT:
            meas = f"{{i['{k}']}}"
        elif t is InfluxType.TIMEINT:
            ts = f"{{i['{k}']}}"
        elif t is InfluxType.TIMESTR:
            if pd:
                ts = f"{{pd.Timestamp(i['{k}']).value}}"
            else:
                ts = f"{{dt_to_int(str_to_dt(i['{k}']))}}"
        elif t is InfluxType.TIMEDT:
            if pd:
                ts = f"{{pd.Timestamp(i['{k}']).value}}"
            else:
                ts = f"{{dt_to_int(i['{k}'])}}"
        elif t is InfluxType.TAG:
            tags.append(f"{k}={{str(i['{k}']).translate(tag_escape)}}")
        elif t in (InfluxType.FLOAT, InfluxType.BOOL):
            fields.append(f"{k}={{i['{k}']}}")
        elif t is InfluxType.INT:
            fields.append(f"{k}={{i['{k}']}}i")
        elif t is InfluxType.DATETIME:
            if pd:
                fields.append(f"{k}={{pd.Timestamp(i['{k}']).value}}i")
            else:
                fields.append(f"{k}={{dt_to_int(i['{k}'])}}i")
        elif t is InfluxType.TIMEDELTA:
            if pd:
                fields.append(f"{k}={{pd.Timedelta(i['{k}']).value}}i")
            else:
                fields.append(f"{k}={{td_to_int(i['{k}'])}}i")
        elif t is InfluxType.STR:
            fields.append(f"{k}=\\\"{{str(i['{k}']).translate(str_escape)}}\\\"")
        elif t is InfluxType.ENUM:
            fields.append(f"{k}=\\\"{{i['{k}'].name}}\\\"")
        else:
            raise TypeError(f"Unknown type: {t!r}")
    extra_tags = extra_tags or {}
    for k, v in extra_tags.items():<commit_msg>Allow InfluxType.ENUM fields to be None<commit_after>import enum
import ciso8601
import time
# noinspection PyUnresolvedReferences
import re  # noqa
from collections import Counter

# noinspection PyUnresolvedReferences
from .common import *  # noqa
from .. import pd


class DataPoint:
    """Dynamically generated datapoint class"""

    def items(self):
        """Returns an iterator over pair of keys and values"""

    def to_dict(self) -> dict:
        """Converts datapoint to a regular dictionary"""

    def to_lineprotocol(self) -> bytes:
        """Returns InfluxDB line protocol representation of datapoint"""


class InfluxType(enum.Enum):
    MEASUREMENT = 0
    TIMEINT = 10
    TIMESTR = 11
    TIMEDT = 12
    TAG = 20
    # Fields (>=30)
    BOOL = 30
    INT = 40
    DATETIME = 41
    TIMEDELTA = 42
    FLOAT = 50
    STR = 60
    ENUM = 61


def str_to_dt(s):
    dt = ciso8601.parse_datetime(s)
    if dt:
        return dt
    raise ValueError(f'Invalid datetime string: {dt!r}')


def dt_to_int(dt):
    if not dt.tzinfo:
        # Assume tz-naive input to be in UTC, not local time
        return int(dt.timestamp() - time.timezone) * 10 ** 9 + dt.microsecond * 1000
    return int(dt.timestamp()) * 10 ** 9 + dt.microsecond * 1000


def td_to_int(td):
    return int(td.total_seconds()) * 10 ** 9 + td.microseconds * 1000


def _gen_parser(schema, meas, rm_none=False, extra_tags=None):
    """Factory of datapoint -> line protocol parsers"""
    tags = []
    fields = []
    ts = None
    meas = meas
    for k, t in schema.items():
        if t is InfluxType.MEASUREMENT:
            meas = f"{{i['{k}']}}"
        elif t is InfluxType.TIMEINT:
            ts = f"{{i['{k}']}}"
        elif t is InfluxType.TIMESTR:
            if pd:
                ts = f"{{pd.Timestamp(i['{k}']).value}}"
            else:
                ts = f"{{dt_to_int(str_to_dt(i['{k}']))}}"
        elif t is InfluxType.TIMEDT:
            if pd:
                ts = f"{{pd.Timestamp(i['{k}']).value}}"
            else:
                ts = f"{{dt_to_int(i['{k}'])}}"
        elif t is InfluxType.TAG:
            tags.append(f"{k}={{str(i['{k}']).translate(tag_escape)}}")
        elif t in (InfluxType.FLOAT, InfluxType.BOOL):
            fields.append(f"{k}={{i['{k}']}}")
        elif t is InfluxType.INT:
            fields.append(f"{k}={{i['{k}']}}i")
        elif t is InfluxType.DATETIME:
            if pd:
                fields.append(f"{k}={{pd.Timestamp(i['{k}']).value}}i")
            else:
                fields.append(f"{k}={{dt_to_int(i['{k}'])}}i")
        elif t is InfluxType.TIMEDELTA:
            if pd:
                fields.append(f"{k}={{pd.Timedelta(i['{k}']).value}}i")
            else:
                fields.append(f"{k}={{td_to_int(i['{k}'])}}i")
        elif t is InfluxType.STR:
            fields.append(f"{k}=\\\"{{str(i['{k}']).translate(str_escape)}}\\\"")
        elif t is InfluxType.ENUM:
            fields.append(f"{k}=\\\"{{getattr(i['{k}'], 'name', None)}}\\\"")
        else:
            raise TypeError(f"Unknown type: {t!r}")
    extra_tags = extra_tags or {}
    for k, v in extra_tags.items():
{'fraction_non_alphanumeric': 0.16209317166560305, 'fraction_numerical': 0.01723037651563497, 'mean_word_length': 2.2181724845995894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3087274', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2132, 'n_words': 506}