<commit_before>import sys

def _print(*args):
    for arg in args:
        sys.stdout.write('%s '%arg)
    sys.stdout.write('\n')

if sys.version.startswith('3.0'):
    PY_VER = '3.x'
    from urllib.parse import quote_plus,parse_qsl
    from urllib.request import urlopen,urlretrieve
    from hashlib import sha1
    new_sha = lambda s: sha1(bytes(s,'utf-8'))
else:
    PY_VER = '2.x'
    from urllib import quote_plus,urlretrieve
    from urllib2 import urlopen, Request, URLError, HTTPError
    from cgi import parse_qsl
    try: # >= 2.5
        from hashlib import sha1 as new_sha
    except ImportError: # < 2.5
        from sha import new as new_sha

def quote(s):
    try:
        return  quote_plus(s.encode('utf-8'),'+.,:|/?&$=')
    except:
        return  quote_plus(s,'+.,:|/?&$=')

def smart_str(s):
    try:
        s = quote(s)
    except:
        pass
    if PY_VER == '2.x':
        return unicode(s).encode('utf-8') # Py2K
    return str(s)#.encode('utf-8') # Py3K

APIPARAMS = ('chxtc', 'chxt', 'chxp', 'chxs', 'chxr', 'chco', 'chtm', 'chld',
    'chts', 'chtt', 'chxl', 'chd', 'chf', 'chg', 'chl', 'chm', 'chp', 'chs',
    'cht', 'chls', 'chdlp', 'chds', 'chbh', 'chdl', 'choe', 'chst')

MARKERS = 'acdostvVhxrRbBDF'

TYPES = ('bvs', 'p3', 'qr', 'lc', 'p', 'bhg', 'pc', 's', 'r', 'rs', 'bvg', 't',
    'v', 'lxy', 'bhs', 'gom', 'ls')

IMGATTRS = ('title','alt','align','border','height','width','ismap','longdesc',
'usemap','id','class','style','lang','xml:lang','onclick','ondblclick','onmousedown',
'onmouseup','onmouseover','onmousemove','onmouseout','onkeypress','onkeydown','onkeyup')

GEO = ('africa','asia','europe','middle_east','south_america','usa','world')<commit_msg>added chma param, fixed py version determination re:issue21<commit_after>import sys

def _print(*args):
    for arg in args:
        sys.stdout.write('%s '%arg)
    sys.stdout.write('\n')

if sys.version.startswith('3'):
    PY_VER = '3.x'
    from urllib.parse import quote_plus,parse_qsl
    from urllib.request import Request, urlopen, urlretrieve
    from hashlib import sha1
    new_sha = lambda s: sha1(bytes(s,'utf-8'))
else:
    PY_VER = '2.x'
    from urllib import quote_plus,urlretrieve
    from urllib2 import urlopen, Request, URLError, HTTPError
    from cgi import parse_qsl
    try: # >= 2.5
        from hashlib import sha1 as new_sha
    except ImportError: # < 2.5
        from sha import new as new_sha

def quote(s):
    try:
        return  quote_plus(s.encode('utf-8'),'+.,:|/?&$=')
    except:
        return  quote_plus(s,'+.,:|/?&$=')

def smart_str(s):
    try:
        s = quote(s)
    except:
        pass
    if PY_VER == '2.x':
        return unicode(s).encode('utf-8') # Py2K
    return str(s)#.encode('utf-8') # Py3K

APIPARAMS = ('chxtc', 'chxt', 'chxp', 'chxs', 'chxr', 'chco', 'chtm', 'chld',
    'chts', 'chtt', 'chxl', 'chd', 'chf', 'chg', 'chl', 'chm', 'chp', 'chs',
    'cht', 'chls', 'chdlp', 'chds', 'chbh', 'chdl', 'choe', 'chst', 'chma')

MARKERS = 'acdostvVhxrRbBDF'

TYPES = ('bvs', 'p3', 'qr', 'lc', 'p', 'bhg', 'pc', 's', 'r', 'rs', 'bvg', 't',
    'v', 'lxy', 'bhs', 'gom', 'ls')

IMGATTRS = ('title','alt','align','border','height','width','ismap','longdesc',
'usemap','id','class','style','lang','xml:lang','onclick','ondblclick','onmousedown',
'onmouseup','onmouseover','onmousemove','onmouseout','onkeypress','onkeydown','onkeyup')

GEO = ('africa','asia','europe','middle_east','south_america','usa','world')
{'fraction_non_alphanumeric': 0.21808203350664357, 'fraction_numerical': 0.011842865395725015, 'mean_word_length': 3.919034090909091, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '485659', 'n_tokens_mistral': 1402, 'n_tokens_neox': 1374, 'n_words': 295}