<commit_before>import os
import sys
import time
import tracemalloc
import unittest

EMPTY_STRING_SIZE = sys.getsizeof(b'')

def get_lineno():
    frame = sys._getframe(1)
    return frame.f_lineno

def allocate_bytes(size):
    filename = __file__
    lineno = get_lineno() + 1
    data = b'x' * (size - EMPTY_STRING_SIZE)
    return data, (filename, lineno)

class TestTracemalloc(unittest.TestCase):
    def setUp(self):
        tracemalloc.enable()

    def tearDown(self):
        tracemalloc.disable()

    def test_get_source(self):
        obj, obj_source = allocate_bytes(12345)
        source = tracemalloc.get_source(obj)
        self.assertEqual(source, obj_source)

    def test_get_process_memory(self):
        obj_size = 12345
        orig = tracemalloc.get_process_memory()
        if orig is None:
            self.skipTest("get_process_memory is not supported")
        obj, obj_source = allocate_bytes(obj_size)
        curr = tracemalloc.get_process_memory()
        self.assertGreaterEqual(curr - orig, obj_size)

    def test_get_stats(self):
        total = 0
        count = 0
        objs = []
        for index in range(5):
            size = 1234
            obj, source = allocate_bytes(size)
            objs.append(obj)
            total += size
            count += 1

            stats = tracemalloc.get_stats()
            filename, lineno = source<commit_msg>fix unit test<commit_after>import os
import sys
import time
import tracemalloc
import unittest

EMPTY_STRING_SIZE = sys.getsizeof(b'')

def get_lineno():
    frame = sys._getframe(1)
    return frame.f_lineno

def allocate_bytes(size):
    filename = __file__
    lineno = get_lineno() + 1
    data = b'x' * (size - EMPTY_STRING_SIZE)
    return data, (filename, lineno)

class TestTracemalloc(unittest.TestCase):
    def setUp(self):
        tracemalloc.enable()

    def tearDown(self):
        tracemalloc.disable()

    def test_get_source(self):
        obj, obj_source = allocate_bytes(12345)
        source = tracemalloc.get_source(obj)
        self.assertEqual(source, obj_source)

    def test_get_process_memory(self):
        obj_size = 10 ** 7
        orig = tracemalloc.get_process_memory()
        if orig is None:
            self.skipTest("get_process_memory is not supported")
        obj, obj_source = allocate_bytes(obj_size)
        curr = tracemalloc.get_process_memory()
        self.assertGreaterEqual(curr - orig, obj_size)

    def test_get_stats(self):
        total = 0
        count = 0
        objs = []
        for index in range(5):
            size = 1234
            obj, source = allocate_bytes(size)
            objs.append(obj)
            total += size
            count += 1

            stats = tracemalloc.get_stats()
            filename, lineno = source
{'fraction_non_alphanumeric': 0.08895265423242468, 'fraction_numerical': 0.013629842180774749, 'mean_word_length': 2.4053724053724053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3667693', 'n_tokens_mistral': 909, 'n_tokens_neox': 855, 'n_words': 198}