<commit_before>#!/bin/env python
from __future__ import division, print_function

import os.path
import sys
import time
import pstats
import argparse
import pprint
import runpy
import shlex

from struct import Struct
from hashlib import sha1
from functools import partial
from collections import Counter
from xml.sax.saxutils import escape

version = '0.3'

eprint = partial(print, file=sys.stderr)
epprint = partial(pprint.pprint, stream=sys.stderr)

DEFAULT_WIDTH = 1200
DEFAULT_ROW_HEIGHT = 24
DEFAULT_FONT_SIZE = 12
DEFAULT_THRESHOLD = 0.1
DEFAULT_FORMAT = 'svg'
DEFAULT_LOG_MULT = 1000000

PY2 = sys.version_info[0] == 2
if PY2:
    bstr = lambda r: r
    ustr = lambda r: r
else:
    def bstr(data):
        if type(data) is str:
            data = data.encode('utf-8')
        return data

    def ustr(data):
        if type(data) is bytes:
            data = data.decode('latin1')
        return data


def get_profiler(cpu):
    try:
        from cProfile import Profile
    except ImportError:
        from Profile import Profile

    if cpu:
        return Profile(time.clock)
    else:
        return Profile()


def get_out(out, default=None):
    out = out or default
    if out:
        return open(out, 'w')
    else:
        return sys.stdout


def gen_colors(s, e, size):
    for i in range(size):
        yield (s[0] + (e[0] - s[0]) * i // size,
               s[1] + (e[1] - s[1]) * i // size,
               s[2] + (e[2] - s[2]) * i // size)


COLORS = list(gen_colors((255, 240, 141), (255, 65, 34), 7))
CCOLORS = list(gen_colors((44, 255, 210), (113, 194, 0), 5))
ECOLORS = list(gen_colors((230, 230, 255), (150, 150, 255), 5))
DCOLORS = list(gen_colors((190, 190, 190), (240, 240, 240), 7))
<commit_msg>built-in profile context manager<commit_after>#!/bin/env python
from __future__ import division, print_function

import os.path
import sys
import time
import pstats
import argparse
import pprint
import runpy
import shlex

from struct import Struct
from hashlib import sha1
from functools import partial
from collections import Counter
from contextlib import contextmanager
from xml.sax.saxutils import escape

version = '0.3'

eprint = partial(print, file=sys.stderr)
epprint = partial(pprint.pprint, stream=sys.stderr)

DEFAULT_WIDTH = 1200
DEFAULT_ROW_HEIGHT = 24
DEFAULT_FONT_SIZE = 12
DEFAULT_THRESHOLD = 0.1
DEFAULT_FORMAT = 'svg'
DEFAULT_LOG_MULT = 1000000

PY2 = sys.version_info[0] == 2
if PY2:
    bstr = lambda r: r
    ustr = lambda r: r
else:
    def bstr(data):
        if type(data) is str:
            data = data.encode('utf-8')
        return data

    def ustr(data):
        if type(data) is bytes:
            data = data.decode('latin1')
        return data


def get_profiler(cpu):
    try:
        from cProfile import Profile
    except ImportError:
        from Profile import Profile

    if cpu:
        return Profile(time.clock)
    else:
        return Profile()


@contextmanager
def profile(filename, cpu=False):
    p = get_profiler(cpu)
    p.enable()
    try:
        yield
    finally:
        p.disable()
        p.dump_stats(filename)


def get_out(out, default=None):
    out = out or default
    if out:
        return open(out, 'w')
    else:
        return sys.stdout


def gen_colors(s, e, size):
    for i in range(size):
        yield (s[0] + (e[0] - s[0]) * i // size,
               s[1] + (e[1] - s[1]) * i // size,
               s[2] + (e[2] - s[2]) * i // size)


COLORS = list(gen_colors((255, 240, 141), (255, 65, 34), 7))
CCOLORS = list(gen_colors((44, 255, 210), (113, 194, 0), 5))
ECOLORS = list(gen_colors((230, 230, 255), (150, 150, 255), 5))
DCOLORS = list(gen_colors((190, 190, 190), (240, 240, 240), 7))

{'fraction_non_alphanumeric': 0.11995667479014352, 'fraction_numerical': 0.057405903059842944, 'mean_word_length': 2.8121775025799796, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2519083', 'n_tokens_mistral': 1449, 'n_tokens_neox': 1269, 'n_words': 395}