<commit_before>//
//  Benchmark.swift
//  Attabench
//
//  Copyright © 2017 <NAME>.
//

import Foundation
import SipHash

extension Array {
    var randomElement: Element {
        precondition(count > 0)
        let index = Int(arc4random_uniform(UInt32(count)))
        return self[index]
    }
}

public class BenchmarkTimer {
    var elapsedTime: TimeInterval? = nil

    @inline(never)
    public func measure(_ body: () -> ()) {
        let start = Timestamp()
        body()
        let end = Timestamp()
        elapsedTime = end - start
    }
}

fileprivate class BenchmarkTask<Input> {
    let title: String
    let body: (Input) -> ((BenchmarkTimer) -> Void)?

    init(_ title: String, _ body: @escaping (Input) -> ((BenchmarkTimer) -> Void)?) {
        self.title = title
        self.body = body
    }

    func generate(input: Input) -> ((BenchmarkTimer) -> Void)? {
        return self.body(input)
    }
}

public struct BenchmarkInstanceKey: SipHashable {
    public let benchmark: String
    public let task: String
    public let size: Int

    public init(benchmark: String, task: String, size: Int) {
        self.benchmark = benchmark
        self.task = task
        self.size = size
    }

    public func appendHashes(to hasher: inout SipHasher) {
        hasher.append(benchmark)
        hasher.append(task)
        hasher.append(size)
    }
    
    public static func ==(left: BenchmarkInstanceKey, right: BenchmarkInstanceKey) -> Bool {
        return left.benchmark == right.benchmark && left.task == right.task && left.size == right.size
    }
}

public protocol BenchmarkProtocol {
    var title: String { get }
    var descriptiveTitle: String? { get }
    var descriptiveAmortizedTitle: String? { get }

    var taskTitles: [String] { get }
    func run(_ title: String, _ size: Int) -> TimeInterval?
    func forgetInputs() // FIXME: Move instances out of here
    func forgetInstances() // FIXME: Move instances out of here
}

public class Benchmark<Input>: BenchmarkProtocol {
    public let title: String
    public var descriptiveTitle: String? = nil
    public var descriptiveAmortizedTitle: String? = nil
    
    public private(set) var taskTitles: [String] = []
    private var tasks: [String: BenchmarkTask<Input>] = [:]
    private var instances: [BenchmarkInstanceKey: (BenchmarkTimer) -> Void] = [:]

    private let inputGenerator: (Int) -> Input
    public private(set) var sizes: [Int] = []
    private var inputs: [Int: Input] = [:] // Input size to input data

    public init(title: String, inputGenerator: @escaping (Int) -> Input) {
        self.title = title
        self.inputGenerator = inputGenerator
    }

    public func addTask(title: String, _ body: @escaping (Input) -> ((BenchmarkTimer) -> Void)?) {
        precondition(self.tasks[title] == nil)
        self.taskTitles.append(title)
        self.tasks[title] = BenchmarkTask(title, body)
    }

    private func instance(for key: BenchmarkInstanceKey) -> ((BenchmarkTimer) -> Void)? {
        if let instance = instances[key] { return instance }
        guard let task = tasks[key.task] else { fatalError() }
        let input = self.input(for: key.size)
        let instance = task.generate(input: input)
        instances[key] = instance
        return instance
    }

    public func forgetInputs() {
        self.inputs = [:]
        forgetInstances()
    }

    public func forgetInstances() {
        self.instances = [:]
    }

    private func input(for size: Int) -> Input {
        if let input = inputs[size] { return input }
        let input = inputGenerator(size)
        sizes.append(size)
        inputs[size] = input
        return input
    }

    @discardableResult @inline(never)
    public func run(_ task: String, _ size: Int) -> TimeInterval? {
        let key = BenchmarkInstanceKey(benchmark: title, task: task, size: size)
        guard let instance = self.instance(for: key) else { return nil }
        let start = Timestamp()<commit_msg>Add Benchmark.addSimpleTask.<commit_after>//
//  Benchmark.swift
//  Attabench
//
//  Copyright © 2017 <NAME>.
//

import Foundation
import SipHash

extension Array {
    var randomElement: Element {
        precondition(count > 0)
        let index = Int(arc4random_uniform(UInt32(count)))
        return self[index]
    }
}

public class BenchmarkTimer {
    var elapsedTime: TimeInterval? = nil

    @inline(never)
    public func measure(_ body: () -> ()) {
        let start = Timestamp()
        body()
        let end = Timestamp()
        elapsedTime = end - start
    }
}

fileprivate class BenchmarkTask<Input> {
    let title: String
    let body: (Input) -> ((BenchmarkTimer) -> Void)?

    init(_ title: String, _ body: @escaping (Input) -> ((BenchmarkTimer) -> Void)?) {
        self.title = title
        self.body = body
    }

    func generate(input: Input) -> ((BenchmarkTimer) -> Void)? {
        return self.body(input)
    }
}

public struct BenchmarkInstanceKey: SipHashable {
    public let benchmark: String
    public let task: String
    public let size: Int

    public init(benchmark: String, task: String, size: Int) {
        self.benchmark = benchmark
        self.task = task
        self.size = size
    }

    public func appendHashes(to hasher: inout SipHasher) {
        hasher.append(benchmark)
        hasher.append(task)
        hasher.append(size)
    }
    
    public static func ==(left: BenchmarkInstanceKey, right: BenchmarkInstanceKey) -> Bool {
        return left.benchmark == right.benchmark && left.task == right.task && left.size == right.size
    }
}

public protocol BenchmarkProtocol {
    var title: String { get }
    var descriptiveTitle: String? { get }
    var descriptiveAmortizedTitle: String? { get }

    var taskTitles: [String] { get }
    func run(_ title: String, _ size: Int) -> TimeInterval?
    func forgetInputs() // FIXME: Move instances out of here
    func forgetInstances() // FIXME: Move instances out of here
}

public class Benchmark<Input>: BenchmarkProtocol {
    public let title: String
    public var descriptiveTitle: String? = nil
    public var descriptiveAmortizedTitle: String? = nil
    
    public private(set) var taskTitles: [String] = []
    private var tasks: [String: BenchmarkTask<Input>] = [:]
    private var instances: [BenchmarkInstanceKey: (BenchmarkTimer) -> Void] = [:]

    private let inputGenerator: (Int) -> Input
    public private(set) var sizes: [Int] = []
    private var inputs: [Int: Input] = [:] // Input size to input data

    public init(title: String, inputGenerator: @escaping (Int) -> Input) {
        self.title = title
        self.inputGenerator = inputGenerator
    }

    public func addTask(title: String, _ body: @escaping (Input) -> ((BenchmarkTimer) -> Void)?) {
        precondition(self.tasks[title] == nil)
        self.taskTitles.append(title)
        self.tasks[title] = BenchmarkTask(title, body)
    }

    public func addSimpleTask(title: String, _ body: @escaping (Input) -> Void) {
        self.addTask(title: title) { input in
            return { timer in timer.measure { body(input) } }
        }
    }

    private func instance(for key: BenchmarkInstanceKey) -> ((BenchmarkTimer) -> Void)? {
        if let instance = instances[key] { return instance }
        guard let task = tasks[key.task] else { fatalError() }
        let input = self.input(for: key.size)
        let instance = task.generate(input: input)
        instances[key] = instance
        return instance
    }

    public func forgetInputs() {
        self.inputs = [:]
        forgetInstances()
    }

    public func forgetInstances() {
        self.instances = [:]
    }

    private func input(for size: Int) -> Input {
        if let input = inputs[size] { return input }
        let input = inputGenerator(size)
        sizes.append(size)
        inputs[size] = input
        return input
    }

    @discardableResult @inline(never)
    public func run(_ task: String, _ size: Int) -> TimeInterval? {
        let key = BenchmarkInstanceKey(benchmark: title, task: task, size: size)
        guard let instance = self.instance(for: key) else { return nil }
        let start = Timestamp()
{'fraction_non_alphanumeric': 0.11345872007816317, 'fraction_numerical': 0.0019540791402051783, 'mean_word_length': 2.6379386939138163, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6518669', 'n_tokens_mistral': 2480, 'n_tokens_neox': 2369, 'n_words': 695}