<commit_before>//  Copyright (c) 2015 <NAME>. All rights reserved.

private func log<C: CollectionType, T>(parser: (Parser<C, T>.Function), function: String = __FUNCTION__, file: String = __FILE__, line: Int = __LINE__, column: Int = __COLUMN__) -> Parser<C, T>.Function {
	return log(nil, parser, function: function, file: file, line: line, column: column)
}

private func log<C: CollectionType, T>(message: String?, parser: (Parser<C, T>.Function), function: String = __FUNCTION__, file: String = __FILE__, line: Int = __LINE__, column: Int = __COLUMN__) -> Parser<C, T>.Function {
	return { collection, index in
		let trace = "\(file):\(line):\(column):\(function):" + (message.map { " \($0)" } ?? "")
		if let (tree, rest) = parser(collection, index) {
			println("\(trace) matched in \(index)..<\(rest)")
			return (tree, rest)
		}
		println("\(trace) unmatched at \(index)")
		return nil
	}
}

private let digit: Parser<String, String>.Function = %("0"..."9")<commit_msg>Log the parsed trees.<commit_after>//  Copyright (c) 2015 <NAME>. All rights reserved.

private func log<C: CollectionType, T>(parser: (Parser<C, T>.Function), function: String = __FUNCTION__, file: String = __FILE__, line: Int = __LINE__, column: Int = __COLUMN__) -> Parser<C, T>.Function {
	return log(nil, parser, function: function, file: file, line: line, column: column)
}

private func log<C: CollectionType, T>(message: String?, parser: (Parser<C, T>.Function), function: String = __FUNCTION__, file: String = __FILE__, line: Int = __LINE__, column: Int = __COLUMN__) -> Parser<C, T>.Function {
	return { collection, index in
		let trace = "\(file):\(line):\(column):\(function):" + (message.map { " \($0)" } ?? "")
		if let (tree, rest) = parser(collection, index) {
			println("\(trace) matched in \(index)..<\(rest): \(tree)")
			return (tree, rest)
		}
		println("\(trace) unmatched at \(index)")
		return nil
	}
}

private let digit: Parser<String, String>.Function = %("0"..."9")
{'fraction_non_alphanumeric': 0.18822932521562658, 'fraction_numerical': 0.007102993404363267, 'mean_word_length': 4.586402266288951, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3802823', 'n_tokens_mistral': 661, 'n_tokens_neox': 632, 'n_words': 197}