<commit_before>#!/usr/bin/python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Print out the total number of bytes in each test.

import sys
import re



preludeRegexp = re.compile('INFO -  0 INFO SimpleTest START')
testStartRegexp = re.compile('INFO TEST-START \| (.*)')

inPrelude = True
currTest = None
currSize = 0
testSizes = {}

for l in sys.stdin:
    currSize += 8 * len(l)

    if inPrelude and preludeRegexp.search(l):
        inPrelude = False
        testSizes['prelude'] = currSize
        currSize = 0
        continue

    m = testStartRegexp.search(l)
    if m:
        newTest = m.group(1)
        if currTest:
            testSizes[currTest] = currSize - 8 * len(l)
            currSize = 8 * len(l)
        else:
            # Previous section was the prelude
            assert currSize == 8 * len(l)
        currTest = newTest

for t, sizes in testSizes.iteritems():
    print '{0:10} bytes from test {1}'.format(sizes, t)


print '{0:10} bytes from the end'.format(currSize)



<commit_msg>Don't multiply by 8 for bytes and print out total size<commit_after>#!/usr/bin/python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Print out the total number of bytes in each test.

import sys
import re



preludeRegexp = re.compile('INFO -  0 INFO SimpleTest START')
testStartRegexp = re.compile('INFO TEST-START \| (.*)')

inPrelude = True
currTest = None
currSize = 0
testSizes = {}

for l in sys.stdin:
    currSize += len(l)

    if inPrelude and preludeRegexp.search(l):
        inPrelude = False
        testSizes['prelude'] = currSize
        currSize = 0
        continue

    m = testStartRegexp.search(l)
    if m:
        newTest = m.group(1)
        if currTest:
            testSizes[currTest] = currSize - len(l)
            currSize = len(l)
        else:
            # Previous section was the prelude
            assert currSize == len(l)
        currTest = newTest


totalSize = 0

for t, size in testSizes.iteritems():
    print '{0:10} bytes from test {1}'.format(size, t)
    totalSize += size

totalSize += currSize

print '{0:10} bytes from the end'.format(currSize)

print 'total size of', totalSize, 'bytes.'
{'fraction_non_alphanumeric': 0.10442909386428281, 'fraction_numerical': 0.014628199918732222, 'mean_word_length': 2.547550432276657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2704654', 'n_tokens_mistral': 832, 'n_tokens_neox': 804, 'n_words': 274}