<commit_before>// would've liked a generic typealias here.
public struct Parser<Token, Output> {
    public let parse: (AnyCollection<Token>) throws -> (output: Output, remainder: AnyCollection<Token>)
}

public func satisfy<T>
    (expect: String, condition: (T) -> Bool) -> Parser<T, T> {
    return Parser { input in
        if let next = input.first {
            if condition(next) {
                return (next, input.dropFirst())
            } else {
                throw Error.Mismatch(expect, String(next))
            }
        } else {
            throw Error.EOF
        }
    }
}

public func token<T: Equatable>(_ token: T) -> Parser<T, T> {
    return satisfy(expect: String(token)) { $0 == token }
}

/** Match several tokens in a row, like e.g. a string. */
public func tokens <T: Equatable, C: Collection where C.Iterator.Element == T> (_ xs: C) -> Parser<T,C> {
    let length = xs.count as! Int
    return count(length, any()) >>- { parsedtokens in
        return parsedtokens.elementsEqual(xs) ? pure(xs) : fail(.Mismatch(String(xs), String(parsedtokens)))
    }
}

/** Return whatever the next token is. */
public func any <T> () -> Parser<T,T> {
    return satisfy(expect: "anything") { T in true }
}

/** Try parser, if it fails return 'otherwise' without consuming input. */
public func optional <T,A> (_ p: Parser<T,A>, otherwise: A) -> Parser<T,A> {
    return p <|> pure(otherwise)
}

/** Delay creation of parser until it is needed. */
public func lazy <T,A> (_ f: @autoclosure(escaping)() -> Parser<T,A>) -> Parser<T,A> {
    return Parser { input in try f().parse(input) }
}

/** Apply parser once, then repeat until it fails. Returns an array of the results. */
public func oneOrMore <T,A> (_ p: Parser<T,A>) -> Parser<T,[A]> {
    return extend <^> p <*> optional( lazy(oneOrMore(p)), otherwise: [] )
}

/** Repeat parser until it fails. Returns an array of the results. */
public func zeroOrMore <T,A> (_ p: Parser<T,A>) -> Parser<T,[A]> {
    return optional( oneOrMore(p), otherwise: [] )
}

/** Repeat parser 'n' times. If 'n' == 0 it always succeeds and returns []. */
public func count <T,A> (_ n: Int, _ p: Parser<T,A>) -> Parser<T,[A]> {
    return n == 0 ? pure([]) : extend <^> p <*> count(n-1, p)
}
<commit_msg>Inline xOrMore, improves performance ~40x<commit_after>// would've liked a generic typealias here.
public struct Parser<Token, Output> {
    public let parse: (AnyCollection<Token>) throws -> (output: Output, remainder: AnyCollection<Token>)
}

public func satisfy<T>
    (expect: String, condition: (T) -> Bool) -> Parser<T, T> {
    return Parser { input in
        if let next = input.first {
            if condition(next) {
                return (next, input.dropFirst())
            } else {
                throw Error.Mismatch(expect, String(next))
            }
        } else {
            throw Error.EOF
        }
    }
}

public func token<T: Equatable>(_ token: T) -> Parser<T, T> {
    return satisfy(expect: String(token)) { $0 == token }
}

/** Match several tokens in a row, like e.g. a string. */
public func tokens <T: Equatable, C: Collection where C.Iterator.Element == T> (_ xs: C) -> Parser<T,C> {
    let length = xs.count as! Int
    return count(length, any()) >>- { parsedtokens in
        return parsedtokens.elementsEqual(xs) ? pure(xs) : fail(.Mismatch(String(xs), String(parsedtokens)))
    }
}

/** Return whatever the next token is. */
public func any <T> () -> Parser<T,T> {
    return satisfy(expect: "anything") { T in true }
}

/** Try parser, if it fails return 'otherwise' without consuming input. */
public func optional <T,A> (_ p: Parser<T,A>, otherwise: A) -> Parser<T,A> {
    return p <|> pure(otherwise)
}

/** Delay creation of parser until it is needed. */
public func lazy <T,A> (_ f: @autoclosure(escaping)() -> Parser<T,A>) -> Parser<T,A> {
    return Parser { input in try f().parse(input) }
}

/** Apply parser once, then repeat until it fails. Returns an array of the results. */
public func oneOrMore <T,A> (_ p: Parser<T,A>) -> Parser<T,[A]> {
    return Parser { input in
        var (first, remainder) = try p.parse(input)
        var result = [first]
        while true {
            do {
                let next = try p.parse(remainder)
                result.append(next.output)
                remainder = next.remainder
            } catch {
                return (result, remainder)
            }
        }
    }
}

/** Repeat parser until it fails. Returns an array of the results. */
public func zeroOrMore <T,A> (_ p: Parser<T,A>) -> Parser<T,[A]> {
    return optional( oneOrMore(p), otherwise: [] )
}

/** Repeat parser 'n' times. If 'n' == 0 it always succeeds and returns []. */
public func count <T,A> (_ n: Int, _ p: Parser<T,A>) -> Parser<T,[A]> {
    return Parser { input in
        var input = input
        var results = [A]()
        for _ in 0..<n {
            let (result, remainder) = try p.parse(input)
            results.append(result)
            input = remainder
        }
        return (results, input)
    }
}

{'fraction_non_alphanumeric': 0.15208374481532688, 'fraction_numerical': 0.0017776022121271974, 'mean_word_length': 2.609408410548824, 'pattern_counts': {'":': 0, '<': 62, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '72042', 'n_tokens_mistral': 1656, 'n_tokens_neox': 1604, 'n_words': 520}