<commit_before><?php

defined("_VALID_ACCESS") || die('Direct access forbidden');

PatchUtil::db_add_column('utils_commondata_tree', 'position', 'I4');

reset_sub_arrays('/');

function reset_sub_arrays($name) {
	Utils_CommonDataCommon::reset_array_positions($name);
	$arr = Utils_CommonDataCommon::get_array($name, false, false, true);
	
	foreach ($arr as $k=>$v) {
		reset_sub_arrays($name . '/' .$k);
	}
}

<commit_msg>Corrected patch to exclude empty nodes<commit_after><?php

defined("_VALID_ACCESS") || die('Direct access forbidden');

PatchUtil::db_add_column('utils_commondata_tree', 'position', 'I4');

reset_sub_arrays('/');

function reset_sub_arrays($name) {
	Utils_CommonDataCommon::reset_array_positions($name);
	$arr = Utils_CommonDataCommon::get_array($name, false, false, true);
	
	foreach ($arr as $k=>$v) {
		if (empty($k)) continue;
		reset_sub_arrays($name . '/' .$k);
	}
}


{'fraction_non_alphanumeric': 0.17633928571428573, 'fraction_numerical': 0.002232142857142857, 'mean_word_length': 7.383177570093458, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4553127', 'n_tokens_mistral': 329, 'n_tokens_neox': 313, 'n_words': 54}