<commit_before>import csv
import io
import logging
import warnings
from urllib.parse import quote as urlquote

import dateutil.parser
import msgpack


log = logging.getLogger(__name__)


def create_url(tmpl, **values):
    """Create url with values

    Args:
        tmpl (str): url template
        values (dict): values for url
    """
    quoted_values = {k: urlquote(str(v)) for k, v in values.items()}
    return tmpl.format(**quoted_values)


def validate_record(record):
    """All input records should contain a column named "time".

    Since we represent records internally as dictionaries, this means
    all record *dictionaries* should contain a key called "time".
    """
    if not any(k in record for k in ("time", b"time")):
        warnings.warn(
            'records should have "time" column to import records properly.',
            category=RuntimeWarning,
        )
    return True


def guess_csv_value(s):
    """Given a (string) CSV value, try to guess its type and return it

    Tries to interpret s. It tries each interpretation in turn, and returns
    the first that succeeds:

    1. As an integer
    2. As a floating point value
    3. If it is "false" or "true" (case insensitive), then as a boolean
    4. If it is "" or "none" or "null" (case insensitive), then as None
    5. As the string itself, unaltered

    Args:
        s (str): value on csv
    Returns:
        Suitable value (int, float, str, bool or None)
    """
    try:
        return int(s)
    except (OverflowError, ValueError):
        try:
            return float(s)
        except (OverflowError, ValueError):
            pass
    lower = s.lower()
    if lower in ("false", "true"):
        return "true" == lower
    elif lower in ("", "none", "null"):
        return None
    else:
        return s


# Convert our dtype names to callables that parse a string into that type
DTYPE_TO_CALLABLE = {
    "bool": bool,
    "float": float,
    "int": int,
    "str": str,
    "guess": guess_csv_value,
}


def merge_dtypes_and_converters(dtypes=None, converters=None):
    """Generate a merged converters dictionary from the user supplied dicts.
    """
    our_converters = {}
    if dtypes is not None:
        try:
            for column_name, dtype in dtypes.items():
                our_converters[column_name] = DTYPE_TO_CALLABLE[dtype]
        except KeyError as e:
            raise ValueError(f'Unrecognized dtype {dtype!r}, must be one of '
                             f'{", ".join(repr(k) for k in sorted(DTYPE_TO_CALLABLE))}')
    if converters is not None:
        for column_name, parse_fn in converters.items():
            our_converters[column_name] = parse_fn
    return our_converters
<commit_msg>No f-strings in some of our supported Pythons<commit_after>import csv
import io
import logging
import warnings
from urllib.parse import quote as urlquote

import dateutil.parser
import msgpack


log = logging.getLogger(__name__)


def create_url(tmpl, **values):
    """Create url with values

    Args:
        tmpl (str): url template
        values (dict): values for url
    """
    quoted_values = {k: urlquote(str(v)) for k, v in values.items()}
    return tmpl.format(**quoted_values)


def validate_record(record):
    """All input records should contain a column named "time".

    Since we represent records internally as dictionaries, this means
    all record *dictionaries* should contain a key called "time".
    """
    if not any(k in record for k in ("time", b"time")):
        warnings.warn(
            'records should have "time" column to import records properly.',
            category=RuntimeWarning,
        )
    return True


def guess_csv_value(s):
    """Given a (string) CSV value, try to guess its type and return it

    Tries to interpret s. It tries each interpretation in turn, and returns
    the first that succeeds:

    1. As an integer
    2. As a floating point value
    3. If it is "false" or "true" (case insensitive), then as a boolean
    4. If it is "" or "none" or "null" (case insensitive), then as None
    5. As the string itself, unaltered

    Args:
        s (str): value on csv
    Returns:
        Suitable value (int, float, str, bool or None)
    """
    try:
        return int(s)
    except (OverflowError, ValueError):
        try:
            return float(s)
        except (OverflowError, ValueError):
            pass
    lower = s.lower()
    if lower in ("false", "true"):
        return "true" == lower
    elif lower in ("", "none", "null"):
        return None
    else:
        return s


# Convert our dtype names to callables that parse a string into that type
DTYPE_TO_CALLABLE = {
    "bool": bool,
    "float": float,
    "int": int,
    "str": str,
    "guess": guess_csv_value,
}


def merge_dtypes_and_converters(dtypes=None, converters=None):
    """Generate a merged converters dictionary from the user supplied dicts.
    """
    our_converters = {}
    if dtypes is not None:
        try:
            for column_name, dtype in dtypes.items():
                our_converters[column_name] = DTYPE_TO_CALLABLE[dtype]
        except KeyError as e:
            raise ValueError('Unrecognized dtype %r, must be one of %s' % (
                dtype, ", ".join(repr(k) for k in sorted(DTYPE_TO_CALLABLE))))
    if converters is not None:
        for column_name, parse_fn in converters.items():
            our_converters[column_name] = parse_fn
    return our_converters

{'fraction_non_alphanumeric': 0.09396709323583181, 'fraction_numerical': 0.0018281535648994515, 'mean_word_length': 2.573481384715872, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5957024', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1595, 'n_words': 614}