<commit_before># -*- coding: utf-8 -*-

from time import mktime
from datetime import datetime
from hashlib import sha256
from bson import json_util
import random, string, json, os

def now():
	now = datetime.utcnow()

	return mktime(now.timetuple()) * 1000 + now.microsecond / 1000

def hash(plain):
	m = sha256()
	m.update(plain)

	return m.hexdigest()

def hash_file(filename, hasher, blocksize = 65536):
	stream = open(filename, "rb")
	buffer = stream.read(blocksize)

	while len(buffer) > 0:
		hasher.update(buffer)
		buffer = stream.read(blocksize)

	stream.close()

	return hasher.hexdigest()

def generate_junk(length, characters = None):
	if characters is None:
		characters = string.ascii_letters + string.digits

	result = []

	for i in range(length):
		index = random.randint(0, len(characters) - 1)
		result.append(characters[index])

	random.shuffle(result)

	return "".join(result)

def to_json(obj):
	return json.dumps(obj, sort_keys = True, default = json_util.default)

def enum(**enums):
    return type('Enum', (), enums)

def strip(text):
	if text is None:
		text = ""

	return text.strip()

def remove_all_files(directory):
	for file in os.listdir(directory):
		path = os.path.join(directory, file)
		os.remove(path)
<commit_msg>implemented utility generator function to read from streams using a buffer<commit_after># -*- coding: utf-8 -*-

from time import mktime
from datetime import datetime
from hashlib import sha256
from bson import json_util
import random, string, json, os

def now():
	now = datetime.utcnow()

	return mktime(now.timetuple()) * 1000 + now.microsecond / 1000

def hash(plain):
	m = sha256()
	m.update(plain)

	return m.hexdigest()

def hash_file(filename, hasher, block_size = 81920):
	stream = open(filename, "rb")

	for bytes in read_from_stream(stream, block_size):
		hasher.update(bytes)

	stream.close()

	return hasher.hexdigest()

def generate_junk(length, characters = None):
	if characters is None:
		characters = string.ascii_letters + string.digits

	result = []

	for i in range(length):
		index = random.randint(0, len(characters) - 1)
		result.append(characters[index])

	random.shuffle(result)

	return "".join(result)

def to_json(obj):
	return json.dumps(obj, sort_keys = True, default = json_util.default)

def enum(**enums):
    return type('Enum', (), enums)

def strip(text):
	if text is None:
		text = ""

	return text.strip()

def remove_all_files(directory):
	for file in os.listdir(directory):
		path = os.path.join(directory, file)
		os.remove(path)

def read_from_stream(stream, block_size = 81920):
	bytes = stream.read(block_size)

	while len(bytes) > 0:
		yield bytes
		bytes = stream.read(block_size)

{'fraction_non_alphanumeric': 0.11942959001782531, 'fraction_numerical': 0.01818181818181818, 'mean_word_length': 3.592471358428805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3117905', 'n_tokens_mistral': 1062, 'n_tokens_neox': 912, 'n_words': 251}