<commit_before># -*- coding: utf-8 -*-
"""
    jinja2.bccache
    ~~~~~~~~~~~~~~

    This module implements the bytecode cache system Jinja is optionally
    using.  This is useful if you have very complex template situations and
    the compiliation of all those templates slow down your application too
    much.

    Situations where this is useful are often forking web applications that
    are initialized on the first request.

    :copyright: (c) 2010 by the Jinja Team.
    :license: BSD.
"""
from os import path, listdir
import sys
import marshal
import tempfile
from six.moves import cPickle as pickle
import fnmatch
try:
    from hashlib import sha1
except ImportError:
    from sha import new as sha1
from jinja2.utils import open_if_exists


# marshal works better on 3.x, one hack less required
if sys.version_info[0] >= 3:
    from io import BytesIO
    marshal_dump = marshal.dump
    marshal_load = marshal.load
else:
    from cStringIO import StringIO as BytesIO

    def marshal_dump(code, f):
        if isinstance(f, file):
            marshal.dump(code, f)
        else:<commit_msg>BytesIO from six.<commit_after># -*- coding: utf-8 -*-
"""
    jinja2.bccache
    ~~~~~~~~~~~~~~

    This module implements the bytecode cache system Jinja is optionally
    using.  This is useful if you have very complex template situations and
    the compiliation of all those templates slow down your application too
    much.

    Situations where this is useful are often forking web applications that
    are initialized on the first request.

    :copyright: (c) 2010 by the Jinja Team.
    :license: BSD.
"""
from os import path, listdir
import sys
import marshal
import tempfile
from six.moves import cPickle as pickle
from six import BytesIO
import fnmatch
try:
    from hashlib import sha1
except ImportError:
    from sha import new as sha1
from jinja2.utils import open_if_exists


# marshal works better on 3.x, one hack less required
if sys.version_info[0] >= 3:
    marshal_dump = marshal.dump
    marshal_load = marshal.load
else:

    def marshal_dump(code, f):
        if isinstance(f, file):
            marshal.dump(code, f)
        else:
{'fraction_non_alphanumeric': 0.07060452238117212, 'fraction_numerical': 0.011075219197046609, 'mean_word_length': 3.169230769230769, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4964712', 'n_tokens_mistral': 646, 'n_tokens_neox': 609, 'n_words': 267}