<commit_before>//
//  CharSequences.swift
//  OysterKit Mac
//
//  Created by <NAME> on 09/07/2014.
//  Copyright (c) 2014 RED When Excited Limited. All rights reserved.
//

import Foundation

public class Keywords : TokenizationState {
    override func stateClassName()->String {
        return "Keywords"
    }
    
    let validStrings : [String]
    
    public init(validStrings:Array<String>){
        self.validStrings = validStrings
        super.init()
    }
    
    public override func scan(operation: TokenizeOperation) {
        var didAdvance = false
        
        if completions(operation.context.consumedCharacters+"\(operation.current)") == nil {
            return
        }
        
        while let allCompletions = completions(operation.context.consumedCharacters+"\(operation.current)") {
            if allCompletions.count == 1 && allCompletions[0] == operation.context.consumedCharacters {
                //Pursue our branches
                emitToken(operation)
                
                scanBranches(operation)
                return
            } else {
                operation.advance()
                didAdvance = true
            }
        }
        
        if (didAdvance){
            scanBranches(operation)
            return
        }
    }
        
    func completions(string:String) -> Array<String>?{
        var allMatches = Array<String>()
        
        for validString in validStrings{
            if validString.hasPrefix(string){
                allMatches.append(validString)
            }
        }
        
        if allMatches.count == 0{
            return nil
        } else {
            return allMatches
        }
    }
    
    
    override func serialize(indentation: String) -> String {
        
        var output = ""
        
        output+="["
        
        var first = true
        for keyword in validStrings {
            if !first {<commit_msg>Some optimisation<commit_after>//
//  CharSequences.swift
//  OysterKit Mac
//
//  Created by <NAME> on 09/07/2014.
//  Copyright (c) 2014 RED When Excited Limited. All rights reserved.
//

import Foundation

public class Keywords : TokenizationState {
    override func stateClassName()->String {
        return "Keywords"
    }
    
    let validStrings : [String]
    
    public init(validStrings:Array<String>){
        self.validStrings = validStrings
        super.init()
    }
    
    public override func scan(operation: TokenizeOperation) {
        let completionsText = operation.context.consumedCharacters + String(operation.current)
        guard let _ = completions(completionsText) else { return }

        var didAdvance = false
        while let allCompletions = completions(operation.context.consumedCharacters + String(operation.current)) {
            if allCompletions.first == operation.context.consumedCharacters {
                emitToken(operation)
                scanBranches(operation)
            } else {
                operation.advance()
                didAdvance = true
            }
        }
        
        if (didAdvance){
            scanBranches(operation)
        }
    }
        
    func completions(string: String) -> [String]? {
        let allMatches = validStrings.filter({ $0.hasPrefix(string) })

        if allMatches.count == 0 {
            return nil
        } else {
            return allMatches
        }
    }
    
    
    override func serialize(indentation: String) -> String {
        
        var output = ""
        
        output+="["
        
        var first = true
        for keyword in validStrings {
            if !first {
{'fraction_non_alphanumeric': 0.08718371837183718, 'fraction_numerical': 0.007975797579757976, 'mean_word_length': 1.6431686046511629, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7134642', 'n_tokens_mistral': 982, 'n_tokens_neox': 939, 'n_words': 217}