<commit_before>import string
import md5
from sys import argv

def MDPrint(str):
    outstr = ''
    for i in str:
        o = ord(i)
        outstr = outstr \
                  + string.hexdigits[(o >> 4) & 0xF] \
                  + string.hexdigits[o & 0xF]
    print outstr,


from time import time

def makestr(start, end):
    result = ''
    for i in range(start, end + 1):
        result = result + chr(i)

    return result


def MDTimeTrial():
    TEST_BLOCK_SIZE = 1000
    TEST_BLOCKS = 10000

    TEST_BYTES = TEST_BLOCK_SIZE * TEST_BLOCKS

    # initialize test data, need temporary string filler

    filsiz = 1 << 8
    filler = makestr(0, filsiz-1)
    data = filler * (TEST_BLOCK_SIZE / filsiz);
    data = data + filler[:(TEST_BLOCK_SIZE % filsiz)]

    del filsiz, filler


    # start timer
    print 'MD5 time trial. Processing', TEST_BYTES, 'characters...'
    t1 = time()

    mdContext = md5.new()

    for i in range(TEST_BLOCKS):
        mdContext.update(data)

    str = mdContext.digest()
    t2 = time()

    MDPrint(str)
    print 'is digest of test input.'
    print 'Seconds to process test input:', t2 - t1
    print 'Characters processed per second:', TEST_BYTES / (t2 - t1)


def MDString(str):
    MDPrint(md5.new(str).digest())
    print '"' + str + '"'


def MDFile(filename):
    f = open(filename, 'rb');
    mdContext = md5.new()

    while 1:
        data = f.read(1024)
        if not data:
            break
        mdContext.update(data)

    MDPrint(mdContext.digest())
    print filename


import sys

def MDFilter():
    mdContext = md5.new()

    while 1:
        data = sys.stdin.read(16)
        if not data:
            break
        mdContext.update(data)

    MDPrint(mdContext.digest())
    print


def MDTestSuite():
    print 'MD5 test suite results:'
    MDString('')
    MDString('a')
    MDString('abc')
    MDString('message digest')
    MDString(makestr(ord('a'), ord('z')))
    MDString(makestr(ord('A'), ord('Z')) \
              + makestr(ord('a'), ord('z')) \
              + makestr(ord('0'), ord('9')))
    MDString((makestr(ord('1'), ord('9')) + '0') * 8)

    # Contents of file foo are "abc"
    MDFile('foo')


# I don't wanna use getopt(), since I want to use the same i/f...
def main():
    if len(argv) == 1:<commit_msg>Remove unneeded continuation chars<commit_after>import string
import md5
from sys import argv

def MDPrint(str):
    outstr = ''
    for i in str:
        o = ord(i)
        outstr = (outstr 
                  + string.hexdigits[(o >> 4) & 0xF] 
                  + string.hexdigits[o & 0xF])
    print outstr,


from time import time

def makestr(start, end):
    result = ''
    for i in range(start, end + 1):
        result = result + chr(i)

    return result


def MDTimeTrial():
    TEST_BLOCK_SIZE = 1000
    TEST_BLOCKS = 10000

    TEST_BYTES = TEST_BLOCK_SIZE * TEST_BLOCKS

    # initialize test data, need temporary string filler

    filsiz = 1 << 8
    filler = makestr(0, filsiz-1)
    data = filler * (TEST_BLOCK_SIZE / filsiz);
    data = data + filler[:(TEST_BLOCK_SIZE % filsiz)]

    del filsiz, filler


    # start timer
    print 'MD5 time trial. Processing', TEST_BYTES, 'characters...'
    t1 = time()

    mdContext = md5.new()

    for i in range(TEST_BLOCKS):
        mdContext.update(data)

    str = mdContext.digest()
    t2 = time()

    MDPrint(str)
    print 'is digest of test input.'
    print 'Seconds to process test input:', t2 - t1
    print 'Characters processed per second:', TEST_BYTES / (t2 - t1)


def MDString(str):
    MDPrint(md5.new(str).digest())
    print '"' + str + '"'


def MDFile(filename):
    f = open(filename, 'rb');
    mdContext = md5.new()

    while 1:
        data = f.read(1024)
        if not data:
            break
        mdContext.update(data)

    MDPrint(mdContext.digest())
    print filename


import sys

def MDFilter():
    mdContext = md5.new()

    while 1:
        data = sys.stdin.read(16)
        if not data:
            break
        mdContext.update(data)

    MDPrint(mdContext.digest())
    print


def MDTestSuite():
    print 'MD5 test suite results:'
    MDString('')
    MDString('a')
    MDString('abc')
    MDString('message digest')
    MDString(makestr(ord('a'), ord('z')))
    MDString(makestr(ord('A'), ord('Z')) 
              + makestr(ord('a'), ord('z')) 
              + makestr(ord('0'), ord('9')))
    MDString((makestr(ord('1'), ord('9')) + '0') * 8)

    # Contents of file foo are "abc"
    MDFile('foo')


# I don't wanna use getopt(), since I want to use the same i/f...
def main():
    if len(argv) == 1:
{'fraction_non_alphanumeric': 0.1358212193534389, 'fraction_numerical': 0.019527012367107834, 'mean_word_length': 2.316546762589928, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4194307', 'n_tokens_mistral': 1655, 'n_tokens_neox': 1537, 'n_words': 430}