<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import hashlib
import argparse

import os, sys, struct, zlib, datetime

from pprint import pprint

def sizeof_fmt(num, suffix='B'):
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f %s%s" % (num, 'Yi', suffix)
    
def list_files( startpath, alias ):
	
	file_list = []
	
	st_mtime = 0
	
	for root, dirs, files in os.walk(startpath):
    
		root = root.replace(startpath, '') 
    
		if root.endswith( "/" ):
			root = root[:-1]
			
		for tmp in files:
			
			f = root + "/" + tmp
			
			stat = os.stat( startpath + f )
			if st_mtime < stat.st_mtime:
				st_mtime = stat.st_mtime
				
			file_list.append( {"PATH" : f , "SIZE" : stat.st_size } )
	
	return [ int( st_mtime ), file_list ]

def write_uint32( f, value ):
	#print( int ( value ) )
	data = struct.pack( 'I', int ( value ) )
	for a in data:
		f.write( "" + str( a ) + "," )

def write_uint64( f, value ):
	#print( int ( value ) )
	data = struct.pack( 'L', int ( value ) )
	for a in data:
		f.write( "" + str( a ) + "," )		

def write_string( f, value ):
	
	write_uint32( f, len( value ) )
	
	for a in value:
		f.write( "'" + str( a ) + "'," )
	
	f.write( "'\\0'," )

def bytes_from_file(filename):
	with open(filename, "rb") as f:
		while True:
			byte = f.read(1)
			if not byte:
				break
			yield(ord(byte))


def check_template( f,  path , f_data ):
	"""
	schreibt den wert direkt ins file f
	"""
	f2 = open( path + f_data["PATH"], "rb")
	data = f2.read(20)
	f2.close()
	
	template = 0
	if data.startswith(b"TEMPLATE_V1"):
		template=1
		
	if f_data["PATH"].endswith(".inc"):
		template=1
	
	write_uint32( f, template )
	
	return template
	

def write_uncompressed_file( stats, f, path , f_data ):
	
	f2 = open( path + f_data["PATH"], "rb")
	data = f2.read()
	f2.close()
	
	h = hashlib.md5()
	h.update( data )
	etag = h.hexdigest().upper()
	
	write_uint32( f, 0 ) # nicht komprimiert
	write_string( f, f_data["PATH"] )
	write_string( f, etag )
	check_template( f, path, f_data )	# auf template prüfen
	write_uint32( f, f_data["SIZE"] )
	
	stats["compressed"] += f_data["SIZE"]
	stats["uncompressed"] += f_data["SIZE"]
	
	if sys.version_info[0] == 2:
		for b in data:
			pprint( b )
			#f.write( "" + str( struct.unpack('>B', b)[0] )  + "," )
			f.write( "" + b  + "," )
	else:
		for b in bytes_from_file( path + f_data["PATH"]):
			f.write( "" + str( b ) + "," )
	f.write( "'\\0'," )

def compress_data( data ):
	
	strats = [ 
		{ "strategy": zlib.Z_DEFAULT_STRATEGY,	"size": 0, "data": None},
		{ "strategy": zlib.Z_FILTERED, 			"size": 0, "data": None},
		{ "strategy": zlib.Z_HUFFMAN_ONLY, 		"size": 0, "data": None},
	]
	outs = []
	
	for strat in strats:
		if sys.version_info[0] == 2:
			compress = zlib.compressobj( 9, zlib.DEFLATED, -15 ,9, strat["strategy"] )
		else:
			compress = zlib.compressobj( level=9, method=zlib.DEFLATED, wbits=-15 ,memLevel=9, strategy=strat["strategy"] )
		deflated = compress.compress(data)
		deflated += compress.flush()
		strat["size"] = len( deflated )
		strat["data"] = deflated
	
	smalest = { "strategy": -1, 	"size": len( data ), "data": data}
	for strat in strats:
		if smalest["size"] > strat["size"]:
			smalest = strat
	
	return smalest
	<commit_msg>pack_dir.py Python 2 Test<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import hashlib
import argparse

import os, sys, struct, zlib, datetime

from pprint import pprint

def sizeof_fmt(num, suffix='B'):
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f %s%s" % (num, 'Yi', suffix)
    
def list_files( startpath, alias ):
	
	file_list = []
	
	st_mtime = 0
	
	for root, dirs, files in os.walk(startpath):
    
		root = root.replace(startpath, '') 
    
		if root.endswith( "/" ):
			root = root[:-1]
			
		for tmp in files:
			
			f = root + "/" + tmp
			
			stat = os.stat( startpath + f )
			if st_mtime < stat.st_mtime:
				st_mtime = stat.st_mtime
				
			file_list.append( {"PATH" : f , "SIZE" : stat.st_size } )
	
	return [ int( st_mtime ), file_list ]

def write_uint32( f, value ):
	#print( int ( value ) )
	data = struct.pack( 'I', int ( value ) )
	for a in data:
		f.write( "" + str( a ) + "," )

def write_uint64( f, value ):
	#print( int ( value ) )
	data = struct.pack( 'L', int ( value ) )
	for a in data:
		f.write( "" + str( a ) + "," )		

def write_string( f, value ):
	
	write_uint32( f, len( value ) )
	
	for a in value:
		f.write( "'" + str( a ) + "'," )
	
	f.write( "'\\0'," )

def bytes_from_file(filename):
	with open(filename, "rb") as f:
		while True:
			byte = f.read(1)
			if not byte:
				break
			yield(ord(byte))


def check_template( f,  path , f_data ):
	"""
	schreibt den wert direkt ins file f
	"""
	f2 = open( path + f_data["PATH"], "rb")
	data = f2.read(20)
	f2.close()
	
	template = 0
	if data.startswith(b"TEMPLATE_V1"):
		template=1
		
	if f_data["PATH"].endswith(".inc"):
		template=1
	
	write_uint32( f, template )
	
	return template
	

def write_uncompressed_file( stats, f, path , f_data ):
	
	f2 = open( path + f_data["PATH"], "rb")
	data = f2.read()
	f2.close()
	
	h = hashlib.md5()
	h.update( data )
	etag = h.hexdigest().upper()
	
	write_uint32( f, 0 ) # nicht komprimiert
	write_string( f, f_data["PATH"] )
	write_string( f, etag )
	check_template( f, path, f_data )	# auf template prüfen
	write_uint32( f, f_data["SIZE"] )
	
	stats["compressed"] += f_data["SIZE"]
	stats["uncompressed"] += f_data["SIZE"]
	
	if sys.version_info[0] == 2:
		for b in data:
			f.write( "" + str(ord(b))  + "," )
	else:
		for b in bytes_from_file( path + f_data["PATH"]):
			f.write( "" + str( b ) + "," )
	f.write( "'\\0'," )

def compress_data( data ):
	
	strats = [ 
		{ "strategy": zlib.Z_DEFAULT_STRATEGY,	"size": 0, "data": None},
		{ "strategy": zlib.Z_FILTERED, 			"size": 0, "data": None},
		{ "strategy": zlib.Z_HUFFMAN_ONLY, 		"size": 0, "data": None},
	]
	outs = []
	
	for strat in strats:
		if sys.version_info[0] == 2:
			compress = zlib.compressobj( 9, zlib.DEFLATED, -15 ,9, strat["strategy"] )
		else:
			compress = zlib.compressobj( level=9, method=zlib.DEFLATED, wbits=-15 ,memLevel=9, strategy=strat["strategy"] )
		deflated = compress.compress(data)
		deflated += compress.flush()
		strat["size"] = len( deflated )
		strat["data"] = deflated
	
	smalest = { "strategy": -1, 	"size": len( data ), "data": data}
	for strat in strats:
		if smalest["size"] > strat["size"]:
			smalest = strat
	
	return smalest
	
{'fraction_non_alphanumeric': 0.1841274612956561, 'fraction_numerical': 0.018638208327070496, 'mean_word_length': 3.0847145488029466, 'pattern_counts': {'":': 24, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6841698', 'n_tokens_mistral': 2976, 'n_tokens_neox': 2667, 'n_words': 667}