<commit_before>##
# 16.02.2005, c
import numpy as nm
import scipy as sc
import scipy.linalg as nla
import scipy.sparse as sp
import pdb

import glob, re, time, sys, os
from copy import *
from sets import Set
from types import UnboundMethodType

from getch import getch

import atexit

nm.set_printoptions( threshold = 100 )

##
# 22.09.2005, c
# 24.10.2005
if sys.version[:5] < '2.4.0':
    def sorted( sequence ):
        tmp = copy( sequence )
        tmp.sort()
        return tmp

##
# 26.05.2006, c
def debug():
    pdb.set_trace()

##
# c: 05.06.2006, r: 26.02.2008
level = 0
def output( *argc, **argv ):
    global level
    format = ' %s' * len( argc )
    msg =  format % argc

    if msg.startswith( ' ...' ):
        level -= 1

    print 'sfe:' + ('  ' * level) + msg
<commit_msg>configurable output prefix<commit_after>##
# 16.02.2005, c
import numpy as nm
import scipy as sc
import scipy.linalg as nla
import scipy.sparse as sp
import pdb

import glob, re, time, sys, os
from copy import *
from sets import Set
from types import UnboundMethodType

from getch import getch

import atexit

nm.set_printoptions( threshold = 100 )

##
# 22.09.2005, c
# 24.10.2005
if sys.version[:5] < '2.4.0':
    def sorted( sequence ):
        tmp = copy( sequence )
        tmp.sort()
        return tmp

##
# 26.05.2006, c
def debug():
    pdb.set_trace()

##
# c: 02.04.2008, r: 02.04.2008
prefix = 'sfe:'
def setOutputPrefix( prefix  = 'sfe:' ):
    globals()['prefix'] = prefix

##
# c: 05.06.2006, r: 02.04.2008
level = 0
def output( *argc, **argv ):
    global level
    format = ' %s' * len( argc )
    msg =  format % argc

    if msg.startswith( ' ...' ):
        level -= 1

    print prefix + ('  ' * level) + msg

{'fraction_non_alphanumeric': 0.13995354239256677, 'fraction_numerical': 0.07549361207897794, 'mean_word_length': 2.6273684210526316, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6110371', 'n_tokens_mistral': 736, 'n_tokens_neox': 635, 'n_words': 202}