<commit_before>from __future__ import absolute_import,unicode_literals
'''
A module to isolate python version compatibility filth.
'''
import os
import sys
import time
import base64
import socket
import collections

major = sys.version_info.major
minor = sys.version_info.minor
micro = sys.version_info.micro

majmin = (major,minor)
version = (major,minor,micro)

if version < (3,0,0):
    import select

    import Queue as queue

    from cStringIO import StringIO as BytesIO

    numtypes = (int,long)
    strtypes = (str,unicode)

    sockerrs = (socket.error,)

    def enbase64(s):
        return s.encode('base64')

    def debase64(s):
        return s.decode('base64')

    def isstr(s):
        return type(s) in (str,unicode)

    def iterbytes(byts):
        for c in byts:
            yield(ord(c))

    def makedirs(path,mode=0o777):
        os.makedirs(path,mode=mode)

else:
    import queue
    import builtins

    from io import BytesIO

    numtypes = (int,)
    strtypes = (str,)

    sockerrs = (builtins.ConnectionError,builtins.FileNotFoundError)<commit_msg>added to_int and to_bytes helpers for py27 py34 compat<commit_after>from __future__ import absolute_import,unicode_literals
'''
A module to isolate python version compatibility filth.
'''
import os
import sys
import time
import base64
import socket
import collections

major = sys.version_info.major
minor = sys.version_info.minor
micro = sys.version_info.micro

majmin = (major,minor)
version = (major,minor,micro)

if version < (3,0,0):
    import select

    import Queue as queue

    from cStringIO import StringIO as BytesIO

    numtypes = (int,long)
    strtypes = (str,unicode)

    fmts = {
        1:'B',
        2:'<H',
        4:'<I',
        8:'<Q',
    }

    sockerrs = (socket.error,)

    def enbase64(s):
        return s.encode('base64')

    def debase64(s):
        return s.decode('base64')

    def isstr(s):
        return type(s) in (str,unicode)

    def iterbytes(byts):
        for c in byts:
            yield(ord(c))

    def makedirs(path,mode=0o777):
        os.makedirs(path,mode=mode)

    def to_bytes(valu, size):
        fmt = fmts.get(size)

        if fmt == None:
            raise Exception('to_bytes size not supported: %d' % (size,))

        return struct.pack(fmt,valu)

    def to_int(byts):
        fmt = fmts.get(len(byts))
        if fmt == None:
            raise Exception('to_int size not supported: %d' % (len(byts),))

        return struct.unpack(fmt,byts)[0]

else:

    import queue
    import builtins

    from io import BytesIO

    numtypes = (int,)
    strtypes = (str,)

    sockerrs = (builtins.ConnectionError,builtins.FileNotFoundError)

    def to_bytes(valu, size):
        return valu.to_bytes(size,byteorder='little')

    def to_int(byts):
        return int.from_bytes(byts,'little')
{'fraction_non_alphanumeric': 0.10817477096546864, 'fraction_numerical': 0.015151515151515152, 'mean_word_length': 2.8573369565217392, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5350356', 'n_tokens_mistral': 1006, 'n_tokens_neox': 922, 'n_words': 242}