<commit_before>//
//  AutoComplete.swift
//
//  Created by <NAME> on 9/19/16.
//
//

import Foundation

public protocol Searchable: Hashable {
    var keywords: [String] { get }
}

open class AutoComplete<T: Searchable> {
    lazy var nodes = [Character: AutoComplete<T>]()
    lazy var items = [T]()

    public init() { }

    // MARK: - Insert / index

    public func insert(_ object: T) {
        for string in object.keywords {
            var tokens = tokenize(string)
            var currentIndex = 0
            var maxIndex = tokens.count
            insert(tokens: &tokens, at: &currentIndex, max: &maxIndex, object: object)
        }
    }

    private func insert(tokens: inout [Character],
                        at currentIndex: inout Int,
                        max maxIndex: inout Int,
                        object: T) {
        if currentIndex < maxIndex {
            let current = tokens[currentIndex]
            currentIndex += 1

            if nodes[current] == nil {
                nodes[current] = AutoComplete<T>()
            }

            nodes[current]?.insert(tokens: &tokens, at: &currentIndex, max: &maxIndex, object: object)
        } else {
            items.append(object)
        }
    }

    public func insert(_ set: [T]) {
        for object in set {
            insert(object)
        }
    }

    // MARK: - Search

    public func search(_ string: String) -> [T] {
        var merged: Set<T>?

        for word in string.components(separatedBy: " ") {
            var wordResults = Set<T>()
            var tokens = tokenize(word)
            var maxIndex = tokens.count
            var currentIndex = 0
            find(tokens: &tokens, at: &currentIndex, max: &maxIndex, into: &wordResults)
            if let results = merged {
                merged = results.intersection(wordResults)
            } else {
                merged = wordResults
            }
        }

        if let results = merged {
            return Array(results)
        }
        return []
    }

    private func find(tokens: inout [Character],
                      at currentIndex: inout Int,
                      max maxIndex: inout Int,
                      into results: inout Set<T>) {
        if currentIndex < maxIndex {
            let current = tokens[currentIndex]
            currentIndex += 1
            nodes[current]?.find(tokens: &tokens, at: &currentIndex, max: &maxIndex, into: &results)
        } else {
            insertAll(into: &results)
        }
    }

    func insertAll(into results: inout Set<T>) {
        for t in items {
            results.insert(t)
        }

        for (_, child) in nodes {
            child.insertAll(into: &results)
        }
    }

    private func tokenize(_ string: String) -> [Character] {
        return Array(string.lowercased().characters)
    }
}
<commit_msg>Remove deprecated use of String.characters<commit_after>//
//  AutoComplete.swift
//
//  Created by <NAME> on 9/19/16.
//
//

import Foundation

public protocol Searchable: Hashable {
    var keywords: [String] { get }
}

open class AutoComplete<T: Searchable> {
    lazy var nodes = [Character: AutoComplete<T>]()
    lazy var items = [T]()

    public init() { }

    // MARK: - Insert / index

    public func insert(_ object: T) {
        for string in object.keywords {
            var tokens = tokenize(string)
            var currentIndex = 0
            var maxIndex = tokens.count
            insert(tokens: &tokens, at: &currentIndex, max: &maxIndex, object: object)
        }
    }

    private func insert(tokens: inout [Character],
                        at currentIndex: inout Int,
                        max maxIndex: inout Int,
                        object: T) {
        if currentIndex < maxIndex {
            let current = tokens[currentIndex]
            currentIndex += 1

            if nodes[current] == nil {
                nodes[current] = AutoComplete<T>()
            }

            nodes[current]?.insert(tokens: &tokens, at: &currentIndex, max: &maxIndex, object: object)
        } else {
            items.append(object)
        }
    }

    public func insert(_ set: [T]) {
        for object in set {
            insert(object)
        }
    }

    // MARK: - Search

    public func search(_ string: String) -> [T] {
        var merged: Set<T>?

        for word in string.components(separatedBy: " ") {
            var wordResults = Set<T>()
            var tokens = tokenize(word)
            var maxIndex = tokens.count
            var currentIndex = 0
            find(tokens: &tokens, at: &currentIndex, max: &maxIndex, into: &wordResults)
            if let results = merged {
                merged = results.intersection(wordResults)
            } else {
                merged = wordResults
            }
        }

        if let results = merged {
            return Array(results)
        }
        return []
    }

    private func find(tokens: inout [Character],
                      at currentIndex: inout Int,
                      max maxIndex: inout Int,
                      into results: inout Set<T>) {
        if currentIndex < maxIndex {
            let current = tokens[currentIndex]
            currentIndex += 1
            nodes[current]?.find(tokens: &tokens, at: &currentIndex, max: &maxIndex, into: &results)
        } else {
            insertAll(into: &results)
        }
    }

    func insertAll(into results: inout Set<T>) {
        for t in items {
            results.insert(t)
        }

        for (_, child) in nodes {
            child.insertAll(into: &results)
        }
    }

    private func tokenize(_ string: String) -> [Character] {
        return Array(string.lowercased())
    }
}

{'fraction_non_alphanumeric': 0.10373733845616487, 'fraction_numerical': 0.0031435557107928748, 'mean_word_length': 1.5785682125168843, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '921066', 'n_tokens_mistral': 1592, 'n_tokens_neox': 1524, 'n_words': 444}