<commit_before>//
//  CharacterReader.swift
//  SwiftSoup
//
//  Created by <NAME> on 10/10/16.
//  Copyright © 2016 <NAME>.. All rights reserved.
//

import Foundation

/**
 CharacterReader consumes tokens off a string. To replace the old TokenQueue.
 */
public final class CharacterReader {
    public static let EOF: UnicodeScalar = "\u{FFFF}"//65535
    private static let maxCacheLen: Int = 12
    private let input: [UnicodeScalar]
    private let length: Int
    private var pos: Int = 0
    private var mark: Int = 0
    //private let stringCache: Array<String?> // holds reused strings in this doc, to lessen garbage

    public init(_ input: String) {
        self.input = Array(input.unicodeScalars)
        self.length = self.input.count
        //stringCache = Array(repeating:nil, count:512)
    }

    public func getPos() -> Int {
        return self.pos
    }

    public func isEmpty() -> Bool {
        return pos >= length
    }

    public func current() -> UnicodeScalar {
        return (pos >= length) ? CharacterReader.EOF : input[pos]
    }

    @discardableResult
    public func consume() -> UnicodeScalar {
        let val = (pos >= length) ? CharacterReader.EOF : input[pos]
        pos += 1
        return val
    }

    public func unconsume() {
        pos -= 1
    }

    public func advance() {
        pos += 1
    }

    public func markPos() {
        mark = pos
    }

    public func rewindToMark() {
        pos = mark
    }

    public func consumeAsString() -> String {
        let p = pos
        pos+=1
        return String(input[p])
        //return String(input, pos+=1, 1)
    }

    /**
     * Returns the number of characters between the current position and the next instance of the input char
     * @param c scan target
     * @return offset between current position and next instance of target. -1 if not found.
     */
    public func nextIndexOf(_ c: UnicodeScalar) -> Int {
        // doesn't handle scanning for surrogates
        for i in pos..<length {
            if (c == input[i]) {
                return i - pos
            }
        }
        return -1
    }

    /**
     * Returns the number of characters between the current position and the next instance of the input sequence
     *
     * @param seq scan target
     * @return offset between current position and next instance of target. -1 if not found.
     */
    public func nextIndexOf(_ seq: String) -> Int {
        // doesn't handle scanning for surrogates
		if(seq.isEmpty) {return -1}
        let startChar: UnicodeScalar = seq.unicodeScalar(0)
        for var offset in pos..<length {
            // scan to first instance of startchar:
            if (startChar != input[offset]) {
                offset+=1
                while(offset < length && startChar != input[offset]) { offset+=1 }
            }
            var i = offset + 1
            let last = i + seq.unicodeScalars.count-1
            if (offset < length && last <= length) {
                var j = 1
                while i < last && seq.unicodeScalar(j) == input[i] {
                    j+=1
                    i+=1
                }
                // found full sequence
                if (i == last) {
                    return offset - pos
                }
            }
        }
        return -1
    }

    public func consumeTo(_ c: UnicodeScalar) -> String {
        let offset = nextIndexOf(c)
        if (offset != -1) {
            let consumed = cacheString(pos, offset)
            pos += offset
            return consumed
        } else {
            return consumeToEnd()
        }
    }

    public func consumeTo(_ seq: String) -> String {
        let offset = nextIndexOf(seq)
        if (offset != -1) {
            let consumed = cacheString(pos, offset)
            pos += offset
            return consumed
        } else {
            return consumeToEnd()
        }
    }

    public func consumeToAny(_ chars: UnicodeScalar...) -> String {
        return consumeToAny(chars)
    }
    public func consumeToAny(_ chars: [UnicodeScalar]) -> String {
        let start: Int = pos
        let remaining: Int = length
        let val = input
		if(start == 2528) {
			let d = 1
		}
        OUTER: while (pos < remaining) {
			if(pos == 41708) {
				let d = 1
			}
			if chars.contains(val[pos]) {
				break OUTER
			}
//            for c in chars {
//                if (val[pos] == c){
//                    break OUTER
//                }
//            }
            pos += 1
        }

        return pos > start ? cacheString(start, pos-start) : ""
    }

    public func consumeToAnySorted(_ chars: UnicodeScalar...) -> String {
        return consumeToAnySorted(chars)
    }
    public func consumeToAnySorted(_ chars: [UnicodeScalar]) -> String {
        let start = pos
        let remaining = length
        let val = input

        while (pos < remaining) {
            if (chars.binarySearch(chars, val[pos]) >= 0) {
                break
            }
            pos += 1
        }

        return pos > start ? cacheString(start, pos-start) : ""
    }

    public func consumeData() -> String {
        // &, <, null
        let start = pos
        let remaining = length
        let val = input

        while (pos < remaining) {
            let c: UnicodeScalar = val[pos]
            if (c == "&" || c ==  "<" || c ==  TokeniserStateVars.nullScalr) {
                break
            }
            pos += 1
        }

        return pos > start ? cacheString(start, pos-start) : ""
    }

    public func consumeTagName() -> String {
        // '\t', '\n', '\r', '\f', ' ', '/', '>', nullChar
        let start = pos
        let remaining = length
        let val = input

        while (pos < remaining) {
            let c: UnicodeScalar = val[pos]
            if (c == "\t" || c ==  "\n" || c ==  "\r" || c ==  UnicodeScalar.BackslashF || c ==  " " || c ==  "/" || c ==  ">" || c ==  TokeniserStateVars.nullScalr) {
                break
            }
            pos += 1
        }
        return pos > start ? cacheString(start, pos-start) : ""
    }

    public func consumeToEnd() -> String {
        let data = cacheString(pos, length-pos)
        pos = length
        return data
    }

    public func consumeLetterSequence() -> String {
        let start = pos
        while (pos < length) {
            let c: UnicodeScalar = input[pos]
            if ((c >= "A" && c <= "Z") || (c >= "a" && c <= "z") || c.isMemberOfCharacterSet(CharacterSet.letters)) {
                pos += 1
            } else {<commit_msg>performance fix<commit_after>//
//  CharacterReader.swift
//  SwiftSoup
//
//  Created by <NAME> on 10/10/16.
//  Copyright © 2016 <NAME>.. All rights reserved.
//

import Foundation

/**
 CharacterReader consumes tokens off a string. To replace the old TokenQueue.
 */
public final class CharacterReader {
    private let empty = ""
    public static let EOF: UnicodeScalar = "\u{FFFF}"//65535
    private static let maxCacheLen: Int = 12
    private let input: [UnicodeScalar]
    private let length: Int
    private var pos: Int = 0
    private var mark: Int = 0
    //private let stringCache: Array<String?> // holds reused strings in this doc, to lessen garbage

    public init(_ input: String) {
        self.input = Array(input.unicodeScalars)
        self.length = self.input.count
        //stringCache = Array(repeating:nil, count:512)
    }

    public func getPos() -> Int {
        return self.pos
    }

    public func isEmpty() -> Bool {
        return pos >= length
    }

    public func current() -> UnicodeScalar {
        return (pos >= length) ? CharacterReader.EOF : input[pos]
    }

    @discardableResult
    public func consume() -> UnicodeScalar {
        let val = (pos >= length) ? CharacterReader.EOF : input[pos]
        pos += 1
        return val
    }

    public func unconsume() {
        pos -= 1
    }

    public func advance() {
        pos += 1
    }

    public func markPos() {
        mark = pos
    }

    public func rewindToMark() {
        pos = mark
    }

    public func consumeAsString() -> String {
        let p = pos
        pos+=1
        return String(input[p])
        //return String(input, pos+=1, 1)
    }

    /**
     * Returns the number of characters between the current position and the next instance of the input char
     * @param c scan target
     * @return offset between current position and next instance of target. -1 if not found.
     */
    public func nextIndexOf(_ c: UnicodeScalar) -> Int {
        // doesn't handle scanning for surrogates
        for i in pos..<length {
            if (c == input[i]) {
                return i - pos
            }
        }
        return -1
    }

    /**
     * Returns the number of characters between the current position and the next instance of the input sequence
     *
     * @param seq scan target
     * @return offset between current position and next instance of target. -1 if not found.
     */
    public func nextIndexOf(_ seq: String) -> Int {
        // doesn't handle scanning for surrogates
		if(seq.isEmpty) {return -1}
        let startChar: UnicodeScalar = seq.unicodeScalar(0)
        for var offset in pos..<length {
            // scan to first instance of startchar:
            if (startChar != input[offset]) {
                offset+=1
                while(offset < length && startChar != input[offset]) { offset+=1 }
            }
            var i = offset + 1
            let last = i + seq.unicodeScalars.count-1
            if (offset < length && last <= length) {
                var j = 1
                while i < last && seq.unicodeScalar(j) == input[i] {
                    j+=1
                    i+=1
                }
                // found full sequence
                if (i == last) {
                    return offset - pos
                }
            }
        }
        return -1
    }

    public func consumeTo(_ c: UnicodeScalar) -> String {
        let offset = nextIndexOf(c)
        if (offset != -1) {
            let consumed = cacheString(pos, offset)
            pos += offset
            return consumed
        } else {
            return consumeToEnd()
        }
    }

    public func consumeTo(_ seq: String) -> String {
        let offset = nextIndexOf(seq)
        if (offset != -1) {
            let consumed = cacheString(pos, offset)
            pos += offset
            return consumed
        } else {
            return consumeToEnd()
        }
    }

    public func consumeToAny(_ chars: UnicodeScalar...) -> String {
        return consumeToAny(chars)
    }
    public func consumeToAny(_ chars: [UnicodeScalar]) -> String {
        let start: Int = pos
        let remaining: Int = length
        let val = input
		if(start == 2528) {
			let d = 1
		}
        OUTER: while (pos < remaining) {
			if(pos == 41708) {
				let d = 1
			}
			if chars.contains(val[pos]) {
				break OUTER
			}
//            for c in chars {
//                if (val[pos] == c){
//                    break OUTER
//                }
//            }
            pos += 1
        }

        return pos > start ? cacheString(start, pos-start) : ""
    }

    public func consumeToAnySorted(_ chars: UnicodeScalar...) -> String {
        return consumeToAnySorted(chars)
    }
    public func consumeToAnySorted(_ chars: [UnicodeScalar]) -> String {
        let start = pos
        let remaining = length
        let val = input

        while (pos < remaining) {
            if (chars.binarySearch(chars, val[pos]) >= 0) {
                break
            }
            pos += 1
        }

        return pos > start ? cacheString(start, pos-start) : ""
    }

    public func consumeData() -> String {
        // &, <, null
        let start = pos
        let remaining = length
        let val = input

        while (pos < remaining) {
            let c: UnicodeScalar = val[pos]
            if (c == "&" || c ==  "<" || c ==  TokeniserStateVars.nullScalr) {
                break
            }
            pos += 1
        }

        return pos > start ? cacheString(start, pos-start) : ""
    }

    public func consumeTagName() -> String {
        // '\t', '\n', '\r', '\f', ' ', '/', '>', nullChar
        let start = pos
        let remaining = length
        let val = input

        while (pos < remaining) {
            let c: UnicodeScalar = val[pos]
            if (c == "\t" || c ==  "\n" || c ==  "\r" || c ==  UnicodeScalar.BackslashF || c ==  " " || c ==  "/" || c ==  ">" || c ==  TokeniserStateVars.nullScalr) {
                break
            }
            pos += 1
        }
        return pos > start ? cacheString(start, pos-start) : empty
    }

    public func consumeToEnd() -> String {
        let data = cacheString(pos, length-pos)
        pos = length
        return data
    }

    public func consumeLetterSequence() -> String {
        let start = pos
        while (pos < length) {
            let c: UnicodeScalar = input[pos]
            if ((c >= "A" && c <= "Z") || (c >= "a" && c <= "z") || c.isMemberOfCharacterSet(CharacterSet.letters)) {
                pos += 1
            } else {
{'fraction_non_alphanumeric': 0.1135984216117772, 'fraction_numerical': 0.009106085900743664, 'mean_word_length': 1.5368623676612128, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '943335', 'n_tokens_mistral': 3840, 'n_tokens_neox': 3586, 'n_words': 1175}