<commit_before>import os, sys, warnings, operator
import time
import types
import numbers
import inspect
import itertools
import string, fnmatch
import unicodedata
import datetime as dt
from collections import defaultdict
from functools import partial
from contextlib import contextmanager
from distutils.version import LooseVersion as _LooseVersion

from threading import Thread, Event
import numpy as np
import param

import json

try:
    from cyordereddict import OrderedDict
except:
    from collections import OrderedDict

try:
   import __builtin__ as builtins # noqa (compatibility)
except:
   import builtins as builtins   # noqa (compatibility)

# Python3 compatibility
if sys.version_info.major >= 3:
    basestring = str
    unicode = str
    long = int
    cmp = lambda a, b: (a>b)-(a<b)
    generator_types = (zip, range, types.GeneratorType)
    RecursionError = RecursionError if sys.version_info.minor > 4 else RuntimeError # noqa
    _getargspec = inspect.getfullargspec
    get_keywords = operator.attrgetter('varkw')
    LooseVersion = _LooseVersion
else:
    basestring = basestring
    unicode = unicode
    from itertools import izip
    generator_types = (izip, xrange, types.GeneratorType) # noqa
    RecursionError = RuntimeError
    _getargspec = inspect.getargspec
    get_keywords = operator.attrgetter('keywords')

    class LooseVersion(_LooseVersion):
        """
        Subclassed to avoid unicode issues in python2
        """

        def __init__ (self, vstring=None):
            if isinstance(vstring, unicode):
                vstring = str(vstring)
            self.parse(vstring)

        def __cmp__(self, other):
            if isinstance(other, unicode):
                other = str(other)
            if isinstance(other, basestring):
                other = LooseVersion(other)
            return cmp(self.version, other.version)

numpy_version = LooseVersion(np.__version__)
param_version = LooseVersion(param.__version__)

datetime_types = (np.datetime64, dt.datetime, dt.date)
timedelta_types = (np.timedelta64, dt.timedelta,)
arraylike_types = (np.ndarray,)

try:
    import pandas as pd
    pandas_version = LooseVersion(pd.__version__)
    if pandas_version > '0.20.0':
        from pandas.core.dtypes.dtypes import DatetimeTZDtypeType
        from pandas.core.dtypes.generic import ABCSeries, ABCIndexClass
    else:
        from pandas.types.dtypes import DatetimeTZDtypeType
        from pandas.types.dtypes.generic import ABCSeries, ABCIndexClass
    pandas_datetime_types = (pd.Timestamp, DatetimeTZDtypeType, pd.Period)
    pandas_timedelta_types = (pd.Timedelta,)
    datetime_types = datetime_types + pandas_datetime_types
    timedelta_types = timedelta_types + pandas_timedelta_types
    arraylike_types = arraylike_types + (ABCSeries, ABCIndexClass)
    if pandas_version > '0.23.0':
        from pandas.core.dtypes.generic import ABCExtensionArray
        arraylike_types = arraylike_types + (ABCExtensionArray,)
except ImportError:
    pd = None

try:
    import cftime
    cftime_types = (cftime.datetime,)
    datetime_types += cftime_types
except:
    cftime_types = ()
_STANDARD_CALENDARS = set(['standard', 'gregorian', 'proleptic_gregorian'])


class VersionError(Exception):
    "Raised when there is a library version mismatch."
    def __init__(self, msg, version=None, min_version=None, **kwargs):
        self.version = version
        self.min_version = min_version
        super(VersionError, self).__init__(msg, **kwargs)


class Config(param.ParameterizedFunction):
    """
    Set of boolean configuration values to change HoloViews' global
    behavior. Typically used to control warnings relating to
    deprecations or set global parameter such as style 'themes'.
    """

    future_deprecations = param.Boolean(default=False, doc="""
       Whether to warn about future deprecations""")

    style_17 = param.Boolean(default=False, doc="""
       Switch to the default style options used up to (and including)
       the HoloViews 1.7 release.""")

    warn_options_call = param.Boolean(default=True, doc="""
       Whether to warn when the deprecated __call__ options syntax is
       used (the opts method should now be used instead). It is
       recommended that users switch this on to update any uses of
       __call__ as it will be deprecated in future.""")

    image_rtol = param.Number(default=10e-4, doc="""
      The tolerance used to enforce regular sampling for regular,
      gridded data where regular sampling is expected. Expressed as the
      maximal allowable sampling difference between sample
      locations.""")

    def __call__(self, **params):
        self.set_param(**params)
        return self

config = Config()

class HashableJSON(json.JSONEncoder):
    """
    Extends JSONEncoder to generate a hashable string for as many types
    of object as possible including nested objects and objects that are
    not normally hashable. The purpose of this class is to generate
    unique strings that once hashed are suitable for use in memoization
    and other cases where deep equality must be tested without storing
    the entire object.

    By default JSONEncoder supports booleans, numbers, strings, lists,
    tuples and dictionaries. In order to support other types such as
    sets, datetime objects and mutable objects such as pandas Dataframes
    or numpy arrays, HashableJSON has to convert these types to
    datastructures that can normally be represented as JSON.

    Support for other object types may need to be introduced in
    future. By default, unrecognized object types are represented by
    their id.

    One limitation of this approach is that dictionaries with composite
    keys (e.g tuples) are not supported due to the JSON spec.
    """
    string_hashable = (dt.datetime,)
    repr_hashable = ()

    def default(self, obj):
        if isinstance(obj, set):
            return hash(frozenset(obj))
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        if pd and isinstance(obj, (pd.Series, pd.DataFrame)):
            return obj.to_csv().encode('utf-8')
        elif isinstance(obj, self.string_hashable):
            return str(obj)
        elif isinstance(obj, self.repr_hashable):
            return repr(obj)
        try:
            return hash(obj)
        except:
            return id(obj)


def merge_option_dicts(old_opts, new_opts):
    """
    Update the old_opts option dictionary with the options defined in
    new_opts. Instead of a shallow update as would be performed by calling
    old_opts.update(new_opts), this updates the dictionaries of all option
    types separately.

    Given two dictionaries
        old_opts = {'a': {'x': 'old', 'y': 'old'}}
    and
        new_opts = {'a': {'y': 'new', 'z': 'new'}, 'b': {'k': 'new'}}
    this returns a dictionary
        {'a': {'x': 'old', 'y': 'new', 'z': 'new'}, 'b': {'k': 'new'}}
    """
    merged = dict(old_opts)

    for option_type, options in new_opts.items():
        if option_type not in merged:
            merged[option_type] = {}

        merged[option_type].update(options)

    return merged


def merge_options_to_dict(options):
    """
    Given a collection of Option objects or partial option dictionaries,
    merge everything to a single dictionary.
    """
    merged_options = {}
    for obj in options:
        if isinstance(obj,dict):
            new_opts = obj
        else:
            new_opts = {obj.key: obj.kwargs}

        merged_options = merge_option_dicts(merged_options, new_opts)
    return merged_options


def deprecated_opts_signature(args, kwargs):
    """
    Utility to help with the deprecation of the old .opts method signature

    Returns whether opts.apply_groups should be used (as a bool) and the
    corresponding options.
    """
    groups = ['plot','style', 'norm']
    opts = {kw for kw in kwargs if kw not in ('backend', 'clone')}
    apply_groups = False
    options = None
    new_kwargs = {}
    if len(args) > 0 and isinstance(args[0], dict):
        apply_groups = True
        if set(args[0].keys()) <= set(groups):
            new_kwargs = args[0]
        else:
            options = args[0]
    elif opts and opts.issubset(set(groups)):
        apply_groups = True
    elif kwargs.get('options', None) is not None:
        apply_groups = True
    elif not args and not kwargs:
        apply_groups = True

    return apply_groups, options, new_kwargs


class periodic(Thread):<commit_msg>Add support for literal opts format (#3301)<commit_after>import os, sys, warnings, operator
import time
import types
import numbers
import inspect
import itertools
import string, fnmatch
import unicodedata
import datetime as dt
from collections import defaultdict
from functools import partial
from contextlib import contextmanager
from distutils.version import LooseVersion as _LooseVersion

from threading import Thread, Event
import numpy as np
import param

import json

try:
    from cyordereddict import OrderedDict
except:
    from collections import OrderedDict

try:
   import __builtin__ as builtins # noqa (compatibility)
except:
   import builtins as builtins   # noqa (compatibility)

# Python3 compatibility
if sys.version_info.major >= 3:
    basestring = str
    unicode = str
    long = int
    cmp = lambda a, b: (a>b)-(a<b)
    generator_types = (zip, range, types.GeneratorType)
    RecursionError = RecursionError if sys.version_info.minor > 4 else RuntimeError # noqa
    _getargspec = inspect.getfullargspec
    get_keywords = operator.attrgetter('varkw')
    LooseVersion = _LooseVersion
else:
    basestring = basestring
    unicode = unicode
    from itertools import izip
    generator_types = (izip, xrange, types.GeneratorType) # noqa
    RecursionError = RuntimeError
    _getargspec = inspect.getargspec
    get_keywords = operator.attrgetter('keywords')

    class LooseVersion(_LooseVersion):
        """
        Subclassed to avoid unicode issues in python2
        """

        def __init__ (self, vstring=None):
            if isinstance(vstring, unicode):
                vstring = str(vstring)
            self.parse(vstring)

        def __cmp__(self, other):
            if isinstance(other, unicode):
                other = str(other)
            if isinstance(other, basestring):
                other = LooseVersion(other)
            return cmp(self.version, other.version)

numpy_version = LooseVersion(np.__version__)
param_version = LooseVersion(param.__version__)

datetime_types = (np.datetime64, dt.datetime, dt.date)
timedelta_types = (np.timedelta64, dt.timedelta,)
arraylike_types = (np.ndarray,)

try:
    import pandas as pd
    pandas_version = LooseVersion(pd.__version__)
    if pandas_version > '0.20.0':
        from pandas.core.dtypes.dtypes import DatetimeTZDtypeType
        from pandas.core.dtypes.generic import ABCSeries, ABCIndexClass
    else:
        from pandas.types.dtypes import DatetimeTZDtypeType
        from pandas.types.dtypes.generic import ABCSeries, ABCIndexClass
    pandas_datetime_types = (pd.Timestamp, DatetimeTZDtypeType, pd.Period)
    pandas_timedelta_types = (pd.Timedelta,)
    datetime_types = datetime_types + pandas_datetime_types
    timedelta_types = timedelta_types + pandas_timedelta_types
    arraylike_types = arraylike_types + (ABCSeries, ABCIndexClass)
    if pandas_version > '0.23.0':
        from pandas.core.dtypes.generic import ABCExtensionArray
        arraylike_types = arraylike_types + (ABCExtensionArray,)
except ImportError:
    pd = None

try:
    import cftime
    cftime_types = (cftime.datetime,)
    datetime_types += cftime_types
except:
    cftime_types = ()
_STANDARD_CALENDARS = set(['standard', 'gregorian', 'proleptic_gregorian'])


class VersionError(Exception):
    "Raised when there is a library version mismatch."
    def __init__(self, msg, version=None, min_version=None, **kwargs):
        self.version = version
        self.min_version = min_version
        super(VersionError, self).__init__(msg, **kwargs)


class Config(param.ParameterizedFunction):
    """
    Set of boolean configuration values to change HoloViews' global
    behavior. Typically used to control warnings relating to
    deprecations or set global parameter such as style 'themes'.
    """

    future_deprecations = param.Boolean(default=False, doc="""
       Whether to warn about future deprecations""")

    style_17 = param.Boolean(default=False, doc="""
       Switch to the default style options used up to (and including)
       the HoloViews 1.7 release.""")

    warn_options_call = param.Boolean(default=True, doc="""
       Whether to warn when the deprecated __call__ options syntax is
       used (the opts method should now be used instead). It is
       recommended that users switch this on to update any uses of
       __call__ as it will be deprecated in future.""")

    image_rtol = param.Number(default=10e-4, doc="""
      The tolerance used to enforce regular sampling for regular,
      gridded data where regular sampling is expected. Expressed as the
      maximal allowable sampling difference between sample
      locations.""")

    def __call__(self, **params):
        self.set_param(**params)
        return self

config = Config()

class HashableJSON(json.JSONEncoder):
    """
    Extends JSONEncoder to generate a hashable string for as many types
    of object as possible including nested objects and objects that are
    not normally hashable. The purpose of this class is to generate
    unique strings that once hashed are suitable for use in memoization
    and other cases where deep equality must be tested without storing
    the entire object.

    By default JSONEncoder supports booleans, numbers, strings, lists,
    tuples and dictionaries. In order to support other types such as
    sets, datetime objects and mutable objects such as pandas Dataframes
    or numpy arrays, HashableJSON has to convert these types to
    datastructures that can normally be represented as JSON.

    Support for other object types may need to be introduced in
    future. By default, unrecognized object types are represented by
    their id.

    One limitation of this approach is that dictionaries with composite
    keys (e.g tuples) are not supported due to the JSON spec.
    """
    string_hashable = (dt.datetime,)
    repr_hashable = ()

    def default(self, obj):
        if isinstance(obj, set):
            return hash(frozenset(obj))
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        if pd and isinstance(obj, (pd.Series, pd.DataFrame)):
            return obj.to_csv().encode('utf-8')
        elif isinstance(obj, self.string_hashable):
            return str(obj)
        elif isinstance(obj, self.repr_hashable):
            return repr(obj)
        try:
            return hash(obj)
        except:
            return id(obj)


def merge_option_dicts(old_opts, new_opts):
    """
    Update the old_opts option dictionary with the options defined in
    new_opts. Instead of a shallow update as would be performed by calling
    old_opts.update(new_opts), this updates the dictionaries of all option
    types separately.

    Given two dictionaries
        old_opts = {'a': {'x': 'old', 'y': 'old'}}
    and
        new_opts = {'a': {'y': 'new', 'z': 'new'}, 'b': {'k': 'new'}}
    this returns a dictionary
        {'a': {'x': 'old', 'y': 'new', 'z': 'new'}, 'b': {'k': 'new'}}
    """
    merged = dict(old_opts)

    for option_type, options in new_opts.items():
        if option_type not in merged:
            merged[option_type] = {}

        merged[option_type].update(options)

    return merged


def merge_options_to_dict(options):
    """
    Given a collection of Option objects or partial option dictionaries,
    merge everything to a single dictionary.
    """
    merged_options = {}
    for obj in options:
        if isinstance(obj,dict):
            new_opts = obj
        else:
            new_opts = {obj.key: obj.kwargs}

        merged_options = merge_option_dicts(merged_options, new_opts)
    return merged_options


def deprecated_opts_signature(args, kwargs):
    """
    Utility to help with the deprecation of the old .opts method signature

    Returns whether opts.apply_groups should be used (as a bool) and the
    corresponding options.
    """
    groups = {'plot','style', 'norm'}
    opts = {kw for kw in kwargs if kw not in ('backend', 'clone')}
    apply_groups = False
    options = None
    new_kwargs = {}
    if len(args) > 0 and isinstance(args[0], dict):
        apply_groups = True
        if (not set(args[0]).issubset(groups) and
            all(isinstance(v, dict) and not set(v).issubset(groups)
                for v in args[0].values())):
            apply_groups = False
        elif set(args[0].keys()) <= groups:
            new_kwargs = args[0]
        else:
            options = args[0]
    elif opts and opts.issubset(set(groups)):
        apply_groups = True
    elif kwargs.get('options', None) is not None:
        apply_groups = True
    elif not args and not kwargs:
        apply_groups = True

    return apply_groups, options, new_kwargs


class periodic(Thread):
{'fraction_non_alphanumeric': 0.08248679398618448, 'fraction_numerical': 0.0037150983920589773, 'mean_word_length': 3.0689655172413794, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1741567', 'n_tokens_mistral': 4958, 'n_tokens_neox': 4764, 'n_words': 1646}