<commit_before>//
//  CerealSerialization.swift
//  Cereal
//
//  Created by <NAME> on 27.04.16.
//  Copyright © 2016 Weebly. All rights reserved.
//

import Foundation

// MARK: Types

/// Recursive tree enum that represents data being encoded/decoded
internal indirect enum CoderTreeValue {
    case StringValue(String)
    case IntValue(Int)
    case Int64Value(Int64)
    case DoubleValue(Double)
    case FloatValue(Float)
    case BoolValue(Bool)
    case NSDateValue(NSDate)
    case NSURLValue(NSURL)

    case PairValue(CoderTreeValue, CoderTreeValue)
    case ArrayValue([CoderTreeValue])
    case SubTree([CoderTreeValue])
    case IdentifyingTree(String,[CoderTreeValue])
}

/// Byte header to identify CoderTreeValue in a byte array
internal enum CerealCoderTreeValueType: UInt8 {
    case String = 1
    case Int = 2
    case Int64 = 3
    case Double = 4
    case Float = 5
    case Bool = 6
    case NSDate = 7
    case NSURL = 8

    case Pair = 9
    case Array = 10
    case SubTree = 11
    case IdentifyingTree = 12
}

// MARK: Helpers

/// function takes a value of type `Type` and returns a byte array representation
private func toByteArray<Type>(value: Type) -> [UInt8] {
    var unsafeValue = value
    return withUnsafePointer(&unsafeValue) {
        Array(UnsafeBufferPointer(start: UnsafePointer<UInt8>($0), count: sizeof(Type)))
    }
}

/// function takes an array of bytes and returns a value of type `Type`
private func fromByteArray<Type>(value: [UInt8], _: Type.Type) -> Type {
    return value.withUnsafeBufferPointer {
        return UnsafePointer<Type>($0.baseAddress).memory
    }
}

/// function reads an array of bytes and returns an integer value (Int, Int64 etc)
private func bytesToInteger<Type where Type: IntegerType>(from: [UInt8]) -> Type {
    return from.withUnsafeBufferPointer({
        UnsafePointer<Type>($0.baseAddress).memory
    })
}

/// function returns a number of .StringValue entries inside an array of CoderTreeValue (including subtrees)
private func stringEntriesCount(array: [CoderTreeValue]) -> Int {
    var result = 0
    for item in array {
        result += item.numberOfStringEntries()
    }
    return result
}

/// function returns a byte array in format [length_in_bytes] + [number of array items] + [bytes of CoderTreeValue array]
/// this allows a decoder to set a capacity for an array of CoderTreeValue beign decoded from bytes
private func countCapacityBytes(array: [CoderTreeValue]) -> [UInt8] {
    var stringMap = Dictionary<String, [UInt8]>(minimumCapacity: stringEntriesCount(array))

    return countCapacityBytes(array, stringMap: &stringMap)
}

/// function returns a byte array in format [length_in_bytes] + [number of array items] + [bytes of CoderTreeValue array]
/// this allows a decoder to set a capacity for an array of CoderTreeValue beign decoded from bytes
/// - note: stringMap dictionary is used to improve string encoding speed, see `String.writeToBuffer`
private func countCapacityBytes(array: [CoderTreeValue], inout stringMap: [String: [UInt8]]) -> [UInt8] {
    var result = [UInt8]()

    for item in array {
        item.writeToBuffer(&result, stringMap: &stringMap)
    }

    return toByteArray(result.count) + toByteArray(array.count) + result
}

// MARK: encoding extensions

private extension String {
    func writeToBuffer(inout buffer: [UInt8], inout stringMap: [String: [UInt8]]) {
        // if the string was already decoded, getting from a dictionary is a cheaper operation than getting utf8 view
        if let bytes = stringMap[self] {
            buffer.appendContentsOf(bytes)
            return
        }

        var array = Array(self.utf8)
        array.insertContentsOf(toByteArray(array.count), at: 0)
        buffer.appendContentsOf(array)
        stringMap.updateValue(array, forKey: self)
    }
}

private extension CoderTreeValue {
    func numberOfStringEntries() -> Int {
        switch self {
        case .StringValue, .NSURLValue:
            return 1
        case let .ArrayValue(items):
            return stringEntriesCount(items)

        case let .SubTree(items):
            return stringEntriesCount(items)

        case let .IdentifyingTree(_,items):
            return 1 + stringEntriesCount(items)

        default:
            return 0
        }
    }

    func writeToBuffer(inout buffer: [UInt8], inout stringMap: [String: [UInt8]]) {
        switch self {
            case .StringValue, .NSURLValue:
                self.writeStringBytes(&buffer, stringMap: &stringMap)

            case .IntValue, .Int64Value, .BoolValue:
                self.writeIntBytes(&buffer)

            case .DoubleValue, .FloatValue, .NSDateValue:
                self.writeFloatBytes(&buffer)

            case let .PairValue(key, value):
                buffer.append(CerealCoderTreeValueType.Pair.rawValue)
                key.writeToBuffer(&buffer, stringMap: &stringMap)
                value.writeToBuffer(&buffer, stringMap: &stringMap)

            case let .ArrayValue(array):
                buffer.append(CerealCoderTreeValueType.Array.rawValue)
                buffer.appendContentsOf(countCapacityBytes(array, stringMap: &stringMap))

            case let .SubTree(items):
                buffer.append(CerealCoderTreeValueType.SubTree.rawValue)
                buffer.appendContentsOf(countCapacityBytes(items, stringMap: &stringMap))

            case let .IdentifyingTree(key, items):
                buffer.append(CerealCoderTreeValueType.IdentifyingTree.rawValue)
                CoderTreeValue.StringValue(key).writeToBuffer(&buffer, stringMap: &stringMap)
                buffer.appendContentsOf(countCapacityBytes(items, stringMap: &stringMap))
        }
    }

    func writeStringBytes(inout buffer: [UInt8], inout stringMap: [String: [UInt8]]) {
        switch self {
        case let .StringValue(string):
            buffer.append(CerealCoderTreeValueType.String.rawValue)
            string.writeToBuffer(&buffer, stringMap: &stringMap)
        case let .NSURLValue(url):
            buffer.append(CerealCoderTreeValueType.NSURL.rawValue)
            url.absoluteString.writeToBuffer(&buffer, stringMap: &stringMap)

        default:
            break
        }
    }

    func writeIntBytes(inout buffer: [UInt8]) {
        switch self {
        case let .IntValue(int):
            buffer.append(CerealCoderTreeValueType.Int.rawValue)
            var array = toByteArray(int)
            array.insertContentsOf(toByteArray(array.count), at: 0)
            buffer.appendContentsOf(array)

        case let .Int64Value(int64):
            buffer.append(CerealCoderTreeValueType.Int64.rawValue)
            var array = toByteArray(int64)
            array.insertContentsOf(toByteArray(array.count), at: 0)
            buffer.appendContentsOf(array)

        case let .BoolValue(bool):
            buffer.append(CerealCoderTreeValueType.Bool.rawValue)
            buffer.append(1)<commit_msg>fix bools encoding<commit_after>//
//  CerealSerialization.swift
//  Cereal
//
//  Created by <NAME> on 27.04.16.
//  Copyright © 2016 Weebly. All rights reserved.
//

import Foundation

// MARK: Types

/// Recursive tree enum that represents data being encoded/decoded
internal indirect enum CoderTreeValue {
    case StringValue(String)
    case IntValue(Int)
    case Int64Value(Int64)
    case DoubleValue(Double)
    case FloatValue(Float)
    case BoolValue(Bool)
    case NSDateValue(NSDate)
    case NSURLValue(NSURL)

    case PairValue(CoderTreeValue, CoderTreeValue)
    case ArrayValue([CoderTreeValue])
    case SubTree([CoderTreeValue])
    case IdentifyingTree(String,[CoderTreeValue])
}

/// Byte header to identify CoderTreeValue in a byte array
internal enum CerealCoderTreeValueType: UInt8 {
    case String = 1
    case Int = 2
    case Int64 = 3
    case Double = 4
    case Float = 5
    case Bool = 6
    case NSDate = 7
    case NSURL = 8

    case Pair = 9
    case Array = 10
    case SubTree = 11
    case IdentifyingTree = 12
}

// MARK: Helpers

/// function takes a value of type `Type` and returns a byte array representation
private func toByteArray<Type>(value: Type) -> [UInt8] {
    var unsafeValue = value
    return withUnsafePointer(&unsafeValue) {
        Array(UnsafeBufferPointer(start: UnsafePointer<UInt8>($0), count: sizeof(Type)))
    }
}

/// function takes an array of bytes and returns a value of type `Type`
private func fromByteArray<Type>(value: [UInt8], _: Type.Type) -> Type {
    return value.withUnsafeBufferPointer {
        return UnsafePointer<Type>($0.baseAddress).memory
    }
}

/// function reads an array of bytes and returns an integer value (Int, Int64 etc)
private func bytesToInteger<Type where Type: IntegerType>(from: [UInt8]) -> Type {
    return from.withUnsafeBufferPointer({
        UnsafePointer<Type>($0.baseAddress).memory
    })
}

/// function returns a number of .StringValue entries inside an array of CoderTreeValue (including subtrees)
private func stringEntriesCount(array: [CoderTreeValue]) -> Int {
    var result = 0
    for item in array {
        result += item.numberOfStringEntries()
    }
    return result
}

/// function returns a byte array in format [length_in_bytes] + [number of array items] + [bytes of CoderTreeValue array]
/// this allows a decoder to set a capacity for an array of CoderTreeValue beign decoded from bytes
private func countCapacityBytes(array: [CoderTreeValue]) -> [UInt8] {
    var stringMap = Dictionary<String, [UInt8]>(minimumCapacity: stringEntriesCount(array))

    return countCapacityBytes(array, stringMap: &stringMap)
}

/// function returns a byte array in format [length_in_bytes] + [number of array items] + [bytes of CoderTreeValue array]
/// this allows a decoder to set a capacity for an array of CoderTreeValue beign decoded from bytes
/// - note: stringMap dictionary is used to improve string encoding speed, see `String.writeToBuffer`
private func countCapacityBytes(array: [CoderTreeValue], inout stringMap: [String: [UInt8]]) -> [UInt8] {
    var result = [UInt8]()

    for item in array {
        item.writeToBuffer(&result, stringMap: &stringMap)
    }

    return toByteArray(result.count) + toByteArray(array.count) + result
}

// MARK: encoding extensions

private extension String {
    func writeToBuffer(inout buffer: [UInt8], inout stringMap: [String: [UInt8]]) {
        // if the string was already decoded, getting from a dictionary is a cheaper operation than getting utf8 view
        if let bytes = stringMap[self] {
            buffer.appendContentsOf(bytes)
            return
        }

        var array = Array(self.utf8)
        array.insertContentsOf(toByteArray(array.count), at: 0)
        buffer.appendContentsOf(array)
        stringMap.updateValue(array, forKey: self)
    }
}

private extension CoderTreeValue {
    func numberOfStringEntries() -> Int {
        switch self {
        case .StringValue, .NSURLValue:
            return 1
        case let .ArrayValue(items):
            return stringEntriesCount(items)

        case let .SubTree(items):
            return stringEntriesCount(items)

        case let .IdentifyingTree(_,items):
            return 1 + stringEntriesCount(items)

        default:
            return 0
        }
    }

    func writeToBuffer(inout buffer: [UInt8], inout stringMap: [String: [UInt8]]) {
        switch self {
            case .StringValue, .NSURLValue:
                self.writeStringBytes(&buffer, stringMap: &stringMap)

            case .IntValue, .Int64Value, .BoolValue:
                self.writeIntBytes(&buffer)

            case .DoubleValue, .FloatValue, .NSDateValue:
                self.writeFloatBytes(&buffer)

            case let .PairValue(key, value):
                buffer.append(CerealCoderTreeValueType.Pair.rawValue)
                key.writeToBuffer(&buffer, stringMap: &stringMap)
                value.writeToBuffer(&buffer, stringMap: &stringMap)

            case let .ArrayValue(array):
                buffer.append(CerealCoderTreeValueType.Array.rawValue)
                buffer.appendContentsOf(countCapacityBytes(array, stringMap: &stringMap))

            case let .SubTree(items):
                buffer.append(CerealCoderTreeValueType.SubTree.rawValue)
                buffer.appendContentsOf(countCapacityBytes(items, stringMap: &stringMap))

            case let .IdentifyingTree(key, items):
                buffer.append(CerealCoderTreeValueType.IdentifyingTree.rawValue)
                CoderTreeValue.StringValue(key).writeToBuffer(&buffer, stringMap: &stringMap)
                buffer.appendContentsOf(countCapacityBytes(items, stringMap: &stringMap))
        }
    }

    func writeStringBytes(inout buffer: [UInt8], inout stringMap: [String: [UInt8]]) {
        switch self {
        case let .StringValue(string):
            buffer.append(CerealCoderTreeValueType.String.rawValue)
            string.writeToBuffer(&buffer, stringMap: &stringMap)
        case let .NSURLValue(url):
            buffer.append(CerealCoderTreeValueType.NSURL.rawValue)
            url.absoluteString.writeToBuffer(&buffer, stringMap: &stringMap)

        default:
            break
        }
    }

    func writeIntBytes(inout buffer: [UInt8]) {
        switch self {
        case let .IntValue(int):
            buffer.append(CerealCoderTreeValueType.Int.rawValue)
            var array = toByteArray(int)
            array.insertContentsOf(toByteArray(array.count), at: 0)
            buffer.appendContentsOf(array)

        case let .Int64Value(int64):
            buffer.append(CerealCoderTreeValueType.Int64.rawValue)
            var array = toByteArray(int64)
            array.insertContentsOf(toByteArray(array.count), at: 0)
            buffer.appendContentsOf(array)

        case let .BoolValue(bool):
            buffer.append(CerealCoderTreeValueType.Bool.rawValue)
            buffer.appendContentsOf(toByteArray(1))
{'fraction_non_alphanumeric': 0.09237178566297946, 'fraction_numerical': 0.010486998994397357, 'mean_word_length': 2.8134757600657356, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6696523', 'n_tokens_mistral': 4054, 'n_tokens_neox': 3866, 'n_words': 1057}