<commit_before>from __future__ import print_function, division, absolute_import

import re
import numpy as np
from dateutil.parser import parse as dateparse
from datetime import datetime, date, time, timedelta
from .dispatch import dispatch
from itertoolz import chain
from .coretypes import (int32, int64, float64, bool_, complex128, datetime_,
                        Option, var, from_numpy, Tuple, null,
                        Record, string, Null, DataShape, real, date_, time_,
                        Unit, timedelta_, TimeDelta, object_)
from .predicates import isdimension, isrecord
from .py2help import _strtypes, _inttypes
from .internal_utils import _toposort, groupby
from .util import subclasses<commit_msg>s not z<commit_after>from __future__ import print_function, division, absolute_import

import re
import numpy as np
from dateutil.parser import parse as dateparse
from datetime import datetime, date, time, timedelta
from .dispatch import dispatch
from itertools import chain
from .coretypes import (int32, int64, float64, bool_, complex128, datetime_,
                        Option, var, from_numpy, Tuple, null,
                        Record, string, Null, DataShape, real, date_, time_,
                        Unit, timedelta_, TimeDelta, object_)
from .predicates import isdimension, isrecord
from .py2help import _strtypes, _inttypes
from .internal_utils import _toposort, groupby
from .util import subclasses
{'fraction_non_alphanumeric': 0.05702364394993046, 'fraction_numerical': 0.013908205841446454, 'mean_word_length': 3.5682539682539685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3789977', 'n_tokens_mistral': 386, 'n_tokens_neox': 365, 'n_words': 155}