<commit_before>#!/usr/bin/env lua
local _G,arg,assert,getmetatable,package,require,string,type
    = _G,arg,assert,getmetatable,package,require,string,type

local save_path = package.path
if arg[1] == nil then
	package.path = '' -- force loading crc32.so
end -- else load the crc32.lua

local C32 = require'crc32'
package.path = save_path

assert(_G.crc32==nil, 'do not pollute global env')
assert(type(C32.version)=='string')
assert(type(C32.crc32)=='function')

do
	local crc32=C32.crc32

	assert(crc32(0, 'egil') == 0x901FF815)
	assert(crc32(0, 'eg') == 0x8A8579E6)
	assert(crc32(0x8A8579E6, 'il') == 0x901FF815)

	-- test with null byte<commit_msg>removed path hack from test_crc32.lua<commit_after>#!/usr/bin/env lua
local _G,arg,assert,getmetatable,package,require,string,type
    = _G,arg,assert,getmetatable,package,require,string,type

local C32 = require'crc32'

assert(_G.crc32==nil, 'do not pollute global env')
assert(type(C32.version)=='string')
assert(type(C32.crc32)=='function')

do
	local crc32=C32.crc32

	assert(crc32(0, 'egil') == 0x901FF815)
	assert(crc32(0, 'eg') == 0x8A8579E6)
	assert(crc32(0x8A8579E6, 'il') == 0x901FF815)

	-- test with null byte
{'fraction_non_alphanumeric': 0.1577152600170503, 'fraction_numerical': 0.09803921568627451, 'mean_word_length': 6.246913580246914, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1947247', 'n_tokens_mistral': 555, 'n_tokens_neox': 490, 'n_words': 85}