<commit_before>//  Copyright (c) 2014 <NAME>. All rights reserved.

/// Convenience for describing the types of parser combinators.
///
/// \param Tree  The type of parse tree generated by the parser.
public struct Parser<Tree> {
	/// The type of parser combinators.
	public typealias Function = String -> (Tree, String)?
}


// MARK: - Terminals

/// Returns a parser which parses `string`.
public func literal(string: String) -> Parser<String>.Function {
	return {
		startsWith($0, string) ?
			(string, $0.fromOffset(countElements(string)))
		:	nil
	}
}


/// Returns a parser which parses any character in `interval`.
public func range<I: IntervalType where I.Bound == Character>(interval: I) -> Parser<String>.Function {
	return { string in
		first(string).map { interval.contains($0) ? ("" + [$0], string.fromOffset(1)) : nil } ?? nil
	}
}


// MARK: - Nonterminals

// MARK: Concatenation

/// Parses the concatenation of `left` and `right`, pairing their parse trees.
public func ++ <T, U> (left: Parser<T>.Function, right: Parser<U>.Function) -> Parser<(T, U)>.Function {
	return concatenate(left, right)
}


// MARK: Alternation

/// Parses either `left` or `right`.
public func | <T, U> (left: Parser<T>.Function, right: Parser<U>.Function) -> Parser<Either<T, U>>.Function {
	return {
		left($0).map { (.left($0), $1) } ?? right($0).map { (.right($0), $1) }
	}
}

/// Parses either `left` or `right` and coalesces their trees.
public func | <T> (left: Parser<T>.Function, right: Parser<T>.Function) -> Parser<T>.Function {
	return left | right --> { $0.either(id, id) }
}


// MARK: Repetition

/// Parses `parser` 0 or more times.
public postfix func * <T> (parser: Parser<T>.Function) -> Parser<[T]>.Function {
	return fix { repeat in
		{
			parser($0).map {
				let repeated = repeat($1) ?? ([], $1)
				return ([$0] + repeated.0, repeated.1)
			} ?? ([], $0)
		}
	}
}

/// Parses `parser` 1 or more times.
public postfix func + <T> (parser: Parser<T>.Function) -> Parser<[T]>.Function {
	return parser ++ parser* --> { [$0] + $1 }
}


// MARK: Mapping

/// Returns a parser which maps parse trees into another type.
public func --> <T, U>(parser: Parser<T>.Function, f: T -> U) -> Parser<U>.Function {
	return {
		parser($0).map { (f($0), $1) }
	}
}


// MARK: Private

/// Defines concatenation for use in the `++` operator definitions above.
private func concatenate<T, U>(left: Parser<T>.Function, right: Parser<U>.Function) -> Parser<(T, U)>.Function {
	return {
		left($0).map { x, rest in<commit_msg>Map ignored parses to ().<commit_after>//  Copyright (c) 2014 <NAME>. All rights reserved.

/// Convenience for describing the types of parser combinators.
///
/// \param Tree  The type of parse tree generated by the parser.
public struct Parser<Tree> {
	/// The type of parser combinators.
	public typealias Function = String -> (Tree, String)?
}


// MARK: - Terminals

/// Returns a parser which parses `string`.
public func literal(string: String) -> Parser<String>.Function {
	return {
		startsWith($0, string) ?
			(string, $0.fromOffset(countElements(string)))
		:	nil
	}
}


/// Returns a parser which parses any character in `interval`.
public func range<I: IntervalType where I.Bound == Character>(interval: I) -> Parser<String>.Function {
	return { string in
		first(string).map { interval.contains($0) ? ("" + [$0], string.fromOffset(1)) : nil } ?? nil
	}
}


// MARK: - Nonterminals

// MARK: Concatenation

/// Parses the concatenation of `left` and `right`, pairing their parse trees.
public func ++ <T, U> (left: Parser<T>.Function, right: Parser<U>.Function) -> Parser<(T, U)>.Function {
	return concatenate(left, right)
}


// MARK: Alternation

/// Parses either `left` or `right`.
public func | <T, U> (left: Parser<T>.Function, right: Parser<U>.Function) -> Parser<Either<T, U>>.Function {
	return {
		left($0).map { (.left($0), $1) } ?? right($0).map { (.right($0), $1) }
	}
}

/// Parses either `left` or `right` and coalesces their trees.
public func | <T> (left: Parser<T>.Function, right: Parser<T>.Function) -> Parser<T>.Function {
	return left | right --> { $0.either(id, id) }
}


// MARK: Repetition

/// Parses `parser` 0 or more times.
public postfix func * <T> (parser: Parser<T>.Function) -> Parser<[T]>.Function {
	return fix { repeat in
		{
			parser($0).map {
				let repeated = repeat($1) ?? ([], $1)
				return ([$0] + repeated.0, repeated.1)
			} ?? ([], $0)
		}
	}
}

/// Parses `parser` 1 or more times.
public postfix func + <T> (parser: Parser<T>.Function) -> Parser<[T]>.Function {
	return parser ++ parser* --> { [$0] + $1 }
}


// MARK: Mapping

/// Returns a parser which maps parse trees into another type.
public func --> <T, U>(parser: Parser<T>.Function, f: T -> U) -> Parser<U>.Function {
	return {
		parser($0).map { (f($0), $1) }
	}
}


// MARK: Ignoring input

/// Ignores any parse trees produced by `parser`.
public func ignore<T>(parser: Parser<T>.Function) -> Parser<()>.Function {
	return parser --> const(())
}


// MARK: Private

/// Defines concatenation for use in the `++` operator definitions above.
private func concatenate<T, U>(left: Parser<T>.Function, right: Parser<U>.Function) -> Parser<(T, U)>.Function {
	return {
		left($0).map { x, rest in
{'fraction_non_alphanumeric': 0.18476808551250237, 'fraction_numerical': 0.011834319526627219, 'mean_word_length': 3.4671781756180735, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 98, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '767696', 'n_tokens_mistral': 1888, 'n_tokens_neox': 1778, 'n_words': 537}