<commit_before># adapted from http://hg.python.org/cpython/file/2.7/Lib/getopt.py
# argv:
#    array of arguments
# shortopts:
#    string of single-letter options. options that take a parameter
#    should be follow by ':'
# longopts:
#    array of strings that are long-form options. options that take
#    a parameter should end with '='
# returns @(opts hash, rem_args array, error string)
function getopt($argv, $shortopts, $longopts) {
	$opts = @{}; $rem = @()

	function err($msg) {
		$opts, $rem, $msg
	}

	# ensure these are arrays
	$argv = @($argv)
	$longopts = @($longopts)

	for($i = 0; $i -lt $argv.length; $i++) {
		$arg = $argv[$i]

		if($arg.startswith('--')) {
			$name = $arg.substring(2)

			$longopt = $longopts | ? { $_ -match "^$name=?$" }

			if($longopt) {
				if($longopt.endswith('=')) { # requires arg
					if($i -eq $argv.length - 1) {
						return err "option --$name requires an argument"
					}<commit_msg>getopt: don't try to parse array arguments<commit_after># adapted from http://hg.python.org/cpython/file/2.7/Lib/getopt.py
# argv:
#    array of arguments
# shortopts:
#    string of single-letter options. options that take a parameter
#    should be follow by ':'
# longopts:
#    array of strings that are long-form options. options that take
#    a parameter should end with '='
# returns @(opts hash, remaining_args array, error string)
function getopt($argv, $shortopts, $longopts) {
	$opts = @{}; $rem = @()

	function err($msg) {
		$opts, $rem, $msg
	}

	# ensure these are arrays
	$argv = @($argv)
	$longopts = @($longopts)

	for($i = 0; $i -lt $argv.length; $i++) {
		$arg = $argv[$i]
		# don't try to parse array arguments
		if($arg -is [array]) { $rem += ,$arg; continue }

		if($arg.startswith('--')) {
			$name = $arg.substring(2)

			$longopt = $longopts | ? { $_ -match "^$name=?$" }

			if($longopt) {
				if($longopt.endswith('=')) { # requires arg
					if($i -eq $argv.length - 1) {
						return err "option --$name requires an argument"
					}
{'fraction_non_alphanumeric': 0.176088044022011, 'fraction_numerical': 0.0050025012506253125, 'mean_word_length': 3.23728813559322, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6292768', 'n_tokens_mistral': 778, 'n_tokens_neox': 698, 'n_words': 206}