<commit_before>from __future__ import division

import pandas as pd
import numpy as np
from functools import wraps, partial
import re
import os
from glob import glob
from math import ceil
from toolz import merge, dissoc
from itertools import count
from operator import getitem

from ..compatibility import BytesIO, unicode, range
from ..utils import textblock, file_size
from .. import array as da

from . import core
from .core import (DataFrame, Series, compute, concat, categorize_block,
        tokens, get)
from .shuffle import set_partition

<commit_msg>use apply from compat<commit_after>from __future__ import division

import pandas as pd
import numpy as np
from functools import wraps, partial
import re
import os
from glob import glob
from math import ceil
from toolz import merge, dissoc
from itertools import count
from operator import getitem

from ..compatibility import BytesIO, unicode, range, apply
from ..utils import textblock, file_size
from .. import array as da

from . import core
from .core import (DataFrame, Series, compute, concat, categorize_block,
        tokens, get)
from .shuffle import set_partition


{'fraction_non_alphanumeric': 0.044933920704845816, 'fraction_numerical': 0.0, 'mean_word_length': 4.651741293532338, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5283917', 'n_tokens_mistral': 308, 'n_tokens_neox': 300, 'n_words': 154}