<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
from itertools import chain

try:
    from collections import OrderedDict as odict
except ImportError:
    from ordereddict import OrderedDict as odict

from ._vendor.five import with_metaclass  # NOQA
from ._vendor.six import (string_types, text_type, integer_types, iteritems, itervalues,
                          iterkeys, wraps)  # NOQA

NoneType = type(None)
primitive_types = chain(string_types, integer_types, (float, complex, bool, NoneType))<commit_msg>fix primitive_types<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
from itertools import chain

try:
    from collections import OrderedDict as odict
except ImportError:
    from ordereddict import OrderedDict as odict

from ._vendor.five import with_metaclass, WhateverIO as StringIO  # NOQA
from ._vendor.six import (string_types, text_type, integer_types, iteritems, itervalues,
                          iterkeys, wraps)  # NOQA

NoneType = type(None)
primitive_types = tuple(chain(string_types, integer_types, (float, complex, bool, NoneType)))
{'fraction_non_alphanumeric': 0.07600341588385995, 'fraction_numerical': 0.0017079419299743809, 'mean_word_length': 4.400921658986175, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4070353', 'n_tokens_mistral': 344, 'n_tokens_neox': 339, 'n_words': 109}