<commit_before>//
//  StringScanner.swift
//  cli
//
//  Created by <NAME> on 25/10/2016.
//
//



/// String Scanner Implementation, similar to NSScanner
/// This class holds an internal index integer that gets advanced as the string is scanned
public class StringScanner {
  
  private let string: String
  private var index: Int
  
  private var stringLength: Int {
    return string.characters.count
  }
  
  private var stringIndex: String.Index {
    return string.index(string.startIndex, offsetBy: index)
  }
  
  
  /// Returns the remaining string
  public var remainingString: String {
    return string.substring(from: stringIndex)
  }
  
  
  /// Initialize a StringScanner
  ///
  /// - parameter string: The string to scan
  ///
  /// - returns:  a new StringScanner
  public init(string: String) {
    self.string = string
    self.index = 0
  }
  
  // FIXME: Cross compile PCRE as posix regex does not have non greedy regex
  
  
  /// Peek for a string length
  /// Peek does not advance the internal string index
  ///
  /// - parameter length: the length to return
  ///
  /// - returns: returns a scanner result
  public func peek(length: Int) -> ScannerResult {
    if index >= stringLength {
      return .end
    }
    
    let end = endIndex(forString: string, length: index + length)
    
    let subString = string.substring(with: stringIndex..<end)
    
    return .value(subString)
  }
  
  
  /// Peek until a pattern is found
  /// Peek does not advance the internal string index
  ///
  /// - parameter pattern: a regex pattern
  ///
  /// - returns: returns a scanner result
  public func peek(untilPattern pattern: String) -> ScannerResult {
    return _peek(untilPattern: pattern).0
  }
  
  
  /// Peek until a string is found
  /// Peek does not advance the internal string index
  ///
  /// - parameter search: the string to search
  ///
  /// - returns: returns a scanner result
  public func peek(untilString search: String) -> ScannerResult {
    return _peek(untilString: search).0
  }
  
  
  /// Peek until for a particular string
  /// Peek does not advance the internal string index
  ///
  /// - parameter search: the string to search
  ///
  /// - returns: returns a scanner result
  public func peek(forString search: String) -> ScannerResult {
    return _peek(forString: search).0
  }
  
  
  /// Peek until a range of chacaters is found
  /// Peek does not advance the internal string index
  /// See `CharactersSet` for ready made range of characters
  ///
  /// - parameter range: the range to search
  ///
  /// - returns: returns a scanner result
  public func peek(untilCharacterSet characerSet: CharactersSet) -> ScannerResult {
      return _peek(untilCharacterSet: characerSet).0
  }
  
  
  /// Peek until a range of chacaters is found, return the character found too
  /// Peek does not advance the internal string index
  /// See `CharactersSet` for ready made range of characters
  ///
  /// - parameter range: the range to search
  ///
  /// - returns: returns a scanner result
  public func peek(forCharacterSet characterSet: CharactersSet) -> ScannerResult {
      return _peek(untilCharacterSet: characterSet, includeLast: true).0
  }
  
  
  /// Scan for a string length
  /// Scan advances the internal string index
  ///
  /// - parameter length: the length to scan
  ///
  /// - returns: returns a scanner result
  public func scan(length: Int) -> ScannerResult {
    return peek(length: length).performIfValue {
      index += length
    }
  }
  
  
  /// Scan until a string is found
  /// Scan advances the internal string index
  ///
  /// - parameter search: the string to search
  ///
  /// - returns: returns a scanner result
  public func scan(untilString search: String) -> ScannerResult {
    let res = _peek(untilString: search)
    
    return res.0.performIfValue {
      index = res.1!
    }
  }
  
  
  /// Scan until a regex pattern is found
  /// Scan advances the internal string index
  ///
  /// - parameter search: the regex pattern to search
  ///
  /// - returns: returns a scanner result
  public func scan(untilPattern pattern: String) -> ScannerResult {
    let res = _peek(untilPattern: pattern)
    
    return res.0.performIfValue {
      index = res.1!
    }
  }
  <commit_msg>Fixing bug in scan until<commit_after>//
//  StringScanner.swift
//  cli
//
//  Created by <NAME> on 25/10/2016.
//
//



/// String Scanner Implementation, similar to NSScanner
/// This class holds an internal index integer that gets advanced as the string is scanned
public class StringScanner {
  
  private let string: String
  private var index: Int
  
  private var stringLength: Int {
    return string.characters.count
  }
  
  private var stringIndex: String.Index {
    return string.index(string.startIndex, offsetBy: index)
  }
  
  
  /// Returns the remaining string
  public var remainingString: String {
    return string.substring(from: stringIndex)
  }
  
  
  /// Initialize a StringScanner
  ///
  /// - parameter string: The string to scan
  ///
  /// - returns:  a new StringScanner
  public init(string: String) {
    self.string = string
    self.index = 0
  }
  
  // FIXME: Cross compile PCRE as posix regex does not have non greedy regex
  
  
  /// Peek for a string length
  /// Peek does not advance the internal string index
  ///
  /// - parameter length: the length to return
  ///
  /// - returns: returns a scanner result
  public func peek(length: Int) -> ScannerResult {
    if index >= stringLength {
      return .end
    }
    
    let end = endIndex(forString: string, length: index + length)
    
    let subString = string.substring(with: stringIndex..<end)
    
    return .value(subString)
  }
  
  
  /// Peek until a pattern is found
  /// Peek does not advance the internal string index
  ///
  /// - parameter pattern: a regex pattern
  ///
  /// - returns: returns a scanner result
  public func peek(untilPattern pattern: String) -> ScannerResult {
    return _peek(untilPattern: pattern).0
  }
  
  
  /// Peek until a string is found
  /// Peek does not advance the internal string index
  ///
  /// - parameter search: the string to search
  ///
  /// - returns: returns a scanner result
  public func peek(untilString search: String) -> ScannerResult {
    return _peek(untilString: search).0
  }
  
  
  /// Peek until for a particular string
  /// Peek does not advance the internal string index
  ///
  /// - parameter search: the string to search
  ///
  /// - returns: returns a scanner result
  public func peek(forString search: String) -> ScannerResult {
    return _peek(forString: search).0
  }
  
  
  /// Peek until a range of chacaters is found
  /// Peek does not advance the internal string index
  /// See `CharactersSet` for ready made range of characters
  ///
  /// - parameter range: the range to search
  ///
  /// - returns: returns a scanner result
  public func peek(untilCharacterSet characerSet: CharactersSet) -> ScannerResult {
      return _peek(untilCharacterSet: characerSet).0
  }
  
  
  /// Peek until a range of chacaters is found, return the character found too
  /// Peek does not advance the internal string index
  /// See `CharactersSet` for ready made range of characters
  ///
  /// - parameter range: the range to search
  ///
  /// - returns: returns a scanner result
  public func peek(forCharacterSet characterSet: CharactersSet) -> ScannerResult {
      return _peek(untilCharacterSet: characterSet, includeLast: true).0
  }
  
  
  /// Scan for a string length
  /// Scan advances the internal string index
  ///
  /// - parameter length: the length to scan
  ///
  /// - returns: returns a scanner result
  public func scan(length: Int) -> ScannerResult {
    return peek(length: length).performIfValue {
      index += length
    }
  }
  
  
  /// Scan until a string is found
  /// Scan advances the internal string index
  ///
  /// - parameter search: the string to search
  ///
  /// - returns: returns a scanner result
  public func scan(untilString search: String) -> ScannerResult {
    let res = _peek(untilString: search)
    
    return res.0.performIfValue {
      index += res.1!
    }
  }
  
  
  /// Scan until a regex pattern is found
  /// Scan advances the internal string index
  ///
  /// - parameter search: the regex pattern to search
  ///
  /// - returns: returns a scanner result
  public func scan(untilPattern pattern: String) -> ScannerResult {
    let res = _peek(untilPattern: pattern)
    
    return res.0.performIfValue {
      index = res.1!
    }
  }
  
{'fraction_non_alphanumeric': 0.10184321045263649, 'fraction_numerical': 0.004199720018665422, 'mean_word_length': 2.355381604696673, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6058853', 'n_tokens_mistral': 2500, 'n_tokens_neox': 2429, 'n_words': 902}