<commit_before><?php

/*
	- go through all zip files in the given dirs non recursively
	- read all the dll filenames from those zips
	- create mappings between dll filename and zip filename

	Usage:
	php dllmap.php path0 [ path1 ... ] > dllmapping.json
*/


/*$dirs = array(
	"C:\\tmp\\core_deps\\vc9\\x86",
	"C:\\tmp\\core_deps\\vc11\\x86",
	"C:\\tmp\\core_deps\\vc11\\x64",
);*/

/*$dirs = array(
	"C:\\tmp\\libs",
);*/

$dirs = array();
foreach (array_slice($_SERVER["argv"], 1) as $item) {
	if (file_exists($item) && is_dir($item)) {
		$dirs[] = $item;
	}
}

if (empty($dirs)) {
	echo "Nothing to do\n";
	die;
}


$out = array(
	"vc9" => array(
		"x86" => array(),
		"x64" => array(),
	),
	"vc11" => array(
		"x86" => array(),
		"x64" => array(),
	),
	"vc12" => array(
		"x86" => array(),
		"x64" => array(),
	),
);


foreach ($dirs as $path) {
	$dir = new DirectoryIterator($path);
	foreach ($dir as $fileinfo) {
		if ($fileinfo->isDot() || $fileinfo->isDir()) {
			continue;
		}

		$pathname = $fileinfo->getPathname();
		$filename = $fileinfo->getFilename();

		if (substr($filename, -3) != "zip") {
			continue;
		}

		if (!preg_match(",.*-(vc\d+)-(x\d\d)\.zip,", $filename, $m)) {
			continue;
		}

		$crt = $m[1];
		$arch = $m[2];

		$zip = new ZipArchive();

		$zip->open($pathname);

		$dlls = array();

		for( $i = 0; $i < $zip->numFiles; $i++ ){
			$stat = $zip->statIndex( $i );
			
			if (substr($stat['name'], -3) != "dll") {
				continue;
			}

			$dlls[] = basename($stat['name']);
		} 
		
		$zip->close();
		unset($zip);


		if (!empty($dlls)) {
			$out[$crt][$arch][$filename] = $dlls;
		}
	}
}

echo json_encode($out);

<commit_msg>add vc14 to dll map script<commit_after><?php

/*
	- go through all zip files in the given dirs non recursively
	- read all the dll filenames from those zips
	- create mappings between dll filename and zip filename

	Usage:
	php dllmap.php path0 [ path1 ... ] > dllmapping.json
*/


/*$dirs = array(
	"C:\\tmp\\core_deps\\vc9\\x86",
	"C:\\tmp\\core_deps\\vc11\\x86",
	"C:\\tmp\\core_deps\\vc11\\x64",
);*/

/*$dirs = array(
	"C:\\tmp\\libs",
);*/

$dirs = array();
foreach (array_slice($_SERVER["argv"], 1) as $item) {
	if (file_exists($item) && is_dir($item)) {
		$dirs[] = $item;
	}
}

if (empty($dirs)) {
	echo "Nothing to do\n";
	die;
}


$out = array(
	"vc9" => array(
		"x86" => array(),
		"x64" => array(),
	),
	"vc11" => array(
		"x86" => array(),
		"x64" => array(),
	),
	"vc12" => array(
		"x86" => array(),
		"x64" => array(),
	),
	"vc14" => array(
		"x86" => array(),
		"x64" => array(),
	),
);


foreach ($dirs as $path) {
	$dir = new DirectoryIterator($path);
	foreach ($dir as $fileinfo) {
		if ($fileinfo->isDot() || $fileinfo->isDir()) {
			continue;
		}

		$pathname = $fileinfo->getPathname();
		$filename = $fileinfo->getFilename();

		if (substr($filename, -3) != "zip") {
			continue;
		}

		if (!preg_match(",.*-(vc\d+)-(x\d\d)\.zip,", $filename, $m)) {
			continue;
		}

		$crt = $m[1];
		$arch = $m[2];

		$zip = new ZipArchive();

		$zip->open($pathname);

		$dlls = array();

		for( $i = 0; $i < $zip->numFiles; $i++ ){
			$stat = $zip->statIndex( $i );
			
			if (substr($stat['name'], -3) != "dll") {
				continue;
			}

			$dlls[] = basename($stat['name']);
		} 
		
		$zip->close();
		unset($zip);


		if (!empty($dlls)) {
			$out[$crt][$arch][$filename] = $dlls;
		}
	}
}

echo json_encode($out);


{'fraction_non_alphanumeric': 0.25036949453148094, 'fraction_numerical': 0.02364765001477978, 'mean_word_length': 3.047846889952153, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4174991', 'n_tokens_mistral': 1619, 'n_tokens_neox': 1435, 'n_words': 299}