<commit_before><?php

// export TZ=UTC; find "DIRECTORYNAME" -type d -or -type f -printf '%y %TY-%Tm-%Td %TT %s %d %h %f\n' > "OUTFILENAME"; unset TZ
// cat diskusage-gs.txt | sed -En -e 's/^d/&/p' -e 's/^f.+\.(jpg)$/&/p' | php scripts/process.php ../diskusage-data/test2

if (!isset($_SERVER['argc'])) {
	echo "Must be run from the command line.\n"; exit(1);
}

$args = array(
	'name' => null,
	'filelist' => null,
	'reportdir' => null,
	'timezone' => 'America/New_York',
	'totalsdepth' => 6,
	'top100depth' => 3,
	'maxlinelength' => 1024,
	'notree' => false,
	'delim' => "\x00",
	'ds' => DIRECTORY_SEPARATOR
);

// syntax: php process.php [-tz '<timezone>'] [-d '<delim>'] [-t <totalsdepth>] [-nt (no tree)] [-ds '<directoryseparator>'] [-td <top100depth>] [-n <reportname>] <reportdir> [<filelist>]

$cliargs = array_slice($_SERVER['argv'], 1);
$syntax = "Syntax: php process.php [options] <reportdir> [<filelist>]\nUse -? for help.\n";

while (!is_null($cliarg = array_shift($cliargs))) {
	$shifted = true;
	
	switch ($cliarg) {
		case '/?':
		case '-?':
		case '-h':
		case '--help':
			echo $syntax;
			// TODO: Output help.
			break;
		case '-tz':
			$args['timezone'] = $shifted = array_shift($cliargs);
			break;
		case '-d':
			$args['delim'] = $shifted = array_shift($cliargs);
			break;
		case '-t':
			$args['totalsdepth'] = $shifted = array_shift($cliargs);
			break;
		case '-nt':
			$args['notree'] = true;
			break;
		case '-ds':
			$args['ds'] = $shifted = array_shift($cliargs);
			break;
		case '-td':
			$args['top100depth'] = $shifted = array_shift($cliargs);
			break;
		case '-n':
			$args['name'] = $shifted = array_shift($cliargs);
			break;
		case '-l':
			$args['maxlinelength'] = $shifted = array_shift($cliargs);
			break;
		default:
			$args['reportdir'] = $cliarg;
			$args['filelist'] = array_shift($cliargs);
			$cliargs = array();
	}
	
	if (is_null($shifted)) {
		echo "Missing value after argument $cliarg\n".$syntax; exit(1);
	}
}

if (is_null($args['reportdir'])) {
	echo "reportdir argument is missing\n".$syntax; exit(1);
}

if (!is_null($args['filelist']) && !is_file($args['filelist'])) {
	echo "The <fileslist> does not exist or is not a file.\n"; exit(1);
}

if (!is_dir($args['reportdir'])) {
	echo "The <reportdir> does not exist or is not a directory.\n"; exit(1);
}

if (is_null($args['filelist'])) {
	$args['filelist'] = 'php://stdin';
}

define('COL_TYPE', 0);
define('COL_DATE', 1);
define('COL_TIME', 2);
define('COL_SIZE', 3);
define('COL_DEPTH', 4);
define('COL_PARENT', 5);
define('COL_NAME', 6);

if (!(function_exists("date_default_timezone_set") ? @(date_default_timezone_set($args['timezone'])) : @(putenv("TZ=".$args['timezone'])))) {
	echo "'timezone' config was set to an invalid identifier."; exit(1);
}

$sizeGroups = array(
	array('label' => '1 GB or More', 'size' => 1024 * 1024 * 1024),
	array('label' => '500 MB - 1 GB', 'size' => 1024 * 1024 * 500),
	array('label' => '250 MB - 500 MB', 'size' => 1024 * 1024 * 250),
	array('label' => '125 MB - 250 MB', 'size' => 1024 * 1024 * 125),
	array('label' => '75 MB - 125 MB', 'size' => 1024 * 1024 * 75),
	array('label' => '25 MB - 75 MB', 'size' => 1024 * 1024 * 25),
	array('label' => '10 MB - 25 MB', 'size' => 1024 * 1024 * 10),
	array('label' => '5 MB - 10 MB', 'size' => 1024 * 1024 * 5),
	array('label' => '1 MB - 5 MB', 'size' => 1024 * 1024 * 1),
	array('label' => '500 KB - 1 MB', 'size' => 1024 * 500),
	array('label' => '250 KB - 500 KB', 'size' => 1024 * 250),
	array('label' => '100 KB - 250 KB', 'size' => 1024 * 100),
	array('label' => '50 KB - 100 KB', 'size' => 1024 * 50),
	array('label' => '25 KB - 50 KB', 'size' => 1024 * 25),
	array('label' => '10 KB - 25 KB', 'size' => 1024 * 10),
	array('label' => '5 KB - 10 KB', 'size' => 1024 * 5),
	array('label' => '1 KB - 5 KB', 'size' => 1024 * 1),
	array('label' => 'Less than 1 KB', 'size' => 0)
);

$dateFormat = 'Y-m-d';
$modifiedGroups = array(
	array('label' => '10 Years or More', 'date' => date($dateFormat, strtotime('-10 year'))),
	array('label' => '5 - 10 Years', 'date' => date($dateFormat, strtotime('-5 year'))),
	array('label' => '2 - 5 Years', 'date' => date($dateFormat, strtotime('-2 year'))),
	array('label' => '1 - 2 Years', 'date' => date($dateFormat, strtotime('-1 year'))),
	array('label' => '270 - 365 Days', 'date' => date($dateFormat, strtotime('-270 day'))),
	array('label' => '180 - 270 Days', 'date' => date($dateFormat, strtotime('-180 day'))),
	array('label' => '90 - 180 Days', 'date' => date($dateFormat, strtotime('-90 day'))),
	array('label' => '60 - 90 Days', 'date' => date($dateFormat, strtotime('-60 day'))),
	array('label' => '30 - 60 Days', 'date' => date($dateFormat, strtotime('-30 day'))),
	array('label' => '15 - 30 Days', 'date' => date($dateFormat, strtotime('-15 day'))),
	array('label' => '7 - 15 Days', 'date' => date($dateFormat, strtotime('-7 day'))),
	array('label' => '1 - 7 Days', 'date' => date($dateFormat, strtotime('-1 day'))),
	array('label' => 'Today', 'date' => date($dateFormat)),
	array('label' => 'Future', 'date' => '9999-99-99')
);

if (($fh = fopen($args['filelist'], 'r')) === FALSE) {
	echo "Failed to open <fileslist> for reading.\n"; exit(1);
}

$dirStack = array();
$dirLookup = array();
$errors = array();
$root = null;

// Read in all lines.
while (($line = fgets($fh, $args['maxlinelength'])) !== FALSE) {
	
	// Trim line separator and split line.
	$split = explode($args['delim'], rtrim($line, "\n\r"), 7);
	
	// Validate the line.
	if (count($split) != 7) {
		echo "Invalid Line (".count($split).")\n";
		array_push($errors, "Invalid column count (".count($split)."):" . $split);
	}
	elseif (!preg_match('/^[a-z]$/i', $split[COL_TYPE])) {
		array_push($errors, array('invalidline', COL_TYPE, $split));
	}
	elseif (!preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/', $split[COL_DATE])) {
		array_push($errors, array('invalidline', COL_DATE, $split));<commit_msg>Removed echo from process.php that is no longer necessary.<commit_after><?php

// export TZ=UTC; find "DIRECTORYNAME" -type d -or -type f -printf '%y %TY-%Tm-%Td %TT %s %d %h %f\n' > "OUTFILENAME"; unset TZ
// cat diskusage-gs.txt | sed -En -e 's/^d/&/p' -e 's/^f.+\.(jpg)$/&/p' | php scripts/process.php ../diskusage-data/test2

if (!isset($_SERVER['argc'])) {
	echo "Must be run from the command line.\n"; exit(1);
}

$args = array(
	'name' => null,
	'filelist' => null,
	'reportdir' => null,
	'timezone' => 'America/New_York',
	'totalsdepth' => 6,
	'top100depth' => 3,
	'maxlinelength' => 1024,
	'notree' => false,
	'delim' => "\x00",
	'ds' => DIRECTORY_SEPARATOR
);

// syntax: php process.php [-tz '<timezone>'] [-d '<delim>'] [-t <totalsdepth>] [-nt (no tree)] [-ds '<directoryseparator>'] [-td <top100depth>] [-n <reportname>] <reportdir> [<filelist>]

$cliargs = array_slice($_SERVER['argv'], 1);
$syntax = "Syntax: php process.php [options] <reportdir> [<filelist>]\nUse -? for help.\n";

while (!is_null($cliarg = array_shift($cliargs))) {
	$shifted = true;
	
	switch ($cliarg) {
		case '/?':
		case '-?':
		case '-h':
		case '--help':
			echo $syntax;
			// TODO: Output help.
			break;
		case '-tz':
			$args['timezone'] = $shifted = array_shift($cliargs);
			break;
		case '-d':
			$args['delim'] = $shifted = array_shift($cliargs);
			break;
		case '-t':
			$args['totalsdepth'] = $shifted = array_shift($cliargs);
			break;
		case '-nt':
			$args['notree'] = true;
			break;
		case '-ds':
			$args['ds'] = $shifted = array_shift($cliargs);
			break;
		case '-td':
			$args['top100depth'] = $shifted = array_shift($cliargs);
			break;
		case '-n':
			$args['name'] = $shifted = array_shift($cliargs);
			break;
		case '-l':
			$args['maxlinelength'] = $shifted = array_shift($cliargs);
			break;
		default:
			$args['reportdir'] = $cliarg;
			$args['filelist'] = array_shift($cliargs);
			$cliargs = array();
	}
	
	if (is_null($shifted)) {
		echo "Missing value after argument $cliarg\n".$syntax; exit(1);
	}
}

if (is_null($args['reportdir'])) {
	echo "reportdir argument is missing\n".$syntax; exit(1);
}

if (!is_null($args['filelist']) && !is_file($args['filelist'])) {
	echo "The <fileslist> does not exist or is not a file.\n"; exit(1);
}

if (!is_dir($args['reportdir'])) {
	echo "The <reportdir> does not exist or is not a directory.\n"; exit(1);
}

if (is_null($args['filelist'])) {
	$args['filelist'] = 'php://stdin';
}

define('COL_TYPE', 0);
define('COL_DATE', 1);
define('COL_TIME', 2);
define('COL_SIZE', 3);
define('COL_DEPTH', 4);
define('COL_PARENT', 5);
define('COL_NAME', 6);

if (!(function_exists("date_default_timezone_set") ? @(date_default_timezone_set($args['timezone'])) : @(putenv("TZ=".$args['timezone'])))) {
	echo "'timezone' config was set to an invalid identifier."; exit(1);
}

$sizeGroups = array(
	array('label' => '1 GB or More', 'size' => 1024 * 1024 * 1024),
	array('label' => '500 MB - 1 GB', 'size' => 1024 * 1024 * 500),
	array('label' => '250 MB - 500 MB', 'size' => 1024 * 1024 * 250),
	array('label' => '125 MB - 250 MB', 'size' => 1024 * 1024 * 125),
	array('label' => '75 MB - 125 MB', 'size' => 1024 * 1024 * 75),
	array('label' => '25 MB - 75 MB', 'size' => 1024 * 1024 * 25),
	array('label' => '10 MB - 25 MB', 'size' => 1024 * 1024 * 10),
	array('label' => '5 MB - 10 MB', 'size' => 1024 * 1024 * 5),
	array('label' => '1 MB - 5 MB', 'size' => 1024 * 1024 * 1),
	array('label' => '500 KB - 1 MB', 'size' => 1024 * 500),
	array('label' => '250 KB - 500 KB', 'size' => 1024 * 250),
	array('label' => '100 KB - 250 KB', 'size' => 1024 * 100),
	array('label' => '50 KB - 100 KB', 'size' => 1024 * 50),
	array('label' => '25 KB - 50 KB', 'size' => 1024 * 25),
	array('label' => '10 KB - 25 KB', 'size' => 1024 * 10),
	array('label' => '5 KB - 10 KB', 'size' => 1024 * 5),
	array('label' => '1 KB - 5 KB', 'size' => 1024 * 1),
	array('label' => 'Less than 1 KB', 'size' => 0)
);

$dateFormat = 'Y-m-d';
$modifiedGroups = array(
	array('label' => '10 Years or More', 'date' => date($dateFormat, strtotime('-10 year'))),
	array('label' => '5 - 10 Years', 'date' => date($dateFormat, strtotime('-5 year'))),
	array('label' => '2 - 5 Years', 'date' => date($dateFormat, strtotime('-2 year'))),
	array('label' => '1 - 2 Years', 'date' => date($dateFormat, strtotime('-1 year'))),
	array('label' => '270 - 365 Days', 'date' => date($dateFormat, strtotime('-270 day'))),
	array('label' => '180 - 270 Days', 'date' => date($dateFormat, strtotime('-180 day'))),
	array('label' => '90 - 180 Days', 'date' => date($dateFormat, strtotime('-90 day'))),
	array('label' => '60 - 90 Days', 'date' => date($dateFormat, strtotime('-60 day'))),
	array('label' => '30 - 60 Days', 'date' => date($dateFormat, strtotime('-30 day'))),
	array('label' => '15 - 30 Days', 'date' => date($dateFormat, strtotime('-15 day'))),
	array('label' => '7 - 15 Days', 'date' => date($dateFormat, strtotime('-7 day'))),
	array('label' => '1 - 7 Days', 'date' => date($dateFormat, strtotime('-1 day'))),
	array('label' => 'Today', 'date' => date($dateFormat)),
	array('label' => 'Future', 'date' => '9999-99-99')
);

if (($fh = fopen($args['filelist'], 'r')) === FALSE) {
	echo "Failed to open <fileslist> for reading.\n"; exit(1);
}

$dirStack = array();
$dirLookup = array();
$errors = array();
$root = null;

// Read in all lines.
while (($line = fgets($fh, $args['maxlinelength'])) !== FALSE) {
	
	// Trim line separator and split line.
	$split = explode($args['delim'], rtrim($line, "\n\r"), 7);
	
	// Validate the line.
	if (count($split) != 7) {
		array_push($errors, "Invalid column count (".count($split)."):" . $split);
	}
	elseif (!preg_match('/^[a-z]$/i', $split[COL_TYPE])) {
		array_push($errors, array('invalidline', COL_TYPE, $split));
	}
	elseif (!preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/', $split[COL_DATE])) {
		array_push($errors, array('invalidline', COL_DATE, $split));
{'fraction_non_alphanumeric': 0.24957713125845737, 'fraction_numerical': 0.05564952638700947, 'mean_word_length': 4.223056537102473, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 179, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1635887', 'n_tokens_mistral': 5201, 'n_tokens_neox': 4452, 'n_words': 1167}