<commit_before>//
//  Document.swift
//  BSON
//
//  Created by <NAME> on 19-05-16.
//
//

import Foundation

public protocol BSONArrayProtocol : _ArrayProtocol {}
extension Array : BSONArrayProtocol {}

extension BSONArrayProtocol where Iterator.Element == Document {
    public init(bsonBytes bytes: [UInt8], validating: Bool = false) {
        var array = [Document]()
        var position = 0
        
        documentLoop: while bytes.count >= position + 5 {
            let length = Int(UnsafePointer<Int32>(Array(bytes[position..<position+4])).pointee)
            
            guard length > 0 else {
                // invalid
                break
            }
            
            guard bytes.count >= position + length else {
                break documentLoop
            }
            
            let document = Document(data: Array(bytes[position..<position+length]))
            
            if validating {
                if document.validate() {
                    array.append(document)
                }
            } else {
                array.append(document)
            }
            
            position += length
        }
        
        self.init(array)
    }
}

private enum ElementType : UInt8 {
    case double = 0x01
    case string = 0x02
    case document = 0x03
    case arrayDocument = 0x04
    case binary = 0x05
    case objectId = 0x07
    case boolean = 0x08
    case utcDateTime = 0x09
    case nullValue = 0x0A
    case regex = 0x0B
    case javascriptCode = 0x0D
    case javascriptCodeWithScope = 0x0F
    case int32 = 0x10
    case timestamp = 0x11
    case int64 = 0x12
    case minKey = 0xFF
    case maxKey = 0x7F
}

/// `Document` is a collection type that uses a BSON document as storage.
/// As such, it can be stored in a file or instantiated from BSON data.<commit_msg>Added `bytes` to Array<Document><commit_after>//
//  Document.swift
//  BSON
//
//  Created by <NAME> on 19-05-16.
//
//

import Foundation

public protocol BSONArrayProtocol : _ArrayProtocol {}
extension Array : BSONArrayProtocol {}

extension BSONArrayProtocol where Iterator.Element == Document {
    public init(bsonBytes bytes: [UInt8], validating: Bool = false) {
        var array = [Document]()
        var position = 0
        
        documentLoop: while bytes.count >= position + 5 {
            let length = Int(UnsafePointer<Int32>(Array(bytes[position..<position+4])).pointee)
            
            guard length > 0 else {
                // invalid
                break
            }
            
            guard bytes.count >= position + length else {
                break documentLoop
            }
            
            let document = Document(data: Array(bytes[position..<position+length]))
            
            if validating {
                if document.validate() {
                    array.append(document)
                }
            } else {
                array.append(document)
            }
            
            position += length
        }
        
        self.init(array)
    }
    
    /// The combined data for all documents in the array
    public var bytes: [UInt8] {
        return self.map { $0.bytes }.reduce([], combine: +)
    }
}

private enum ElementType : UInt8 {
    case double = 0x01
    case string = 0x02
    case document = 0x03
    case arrayDocument = 0x04
    case binary = 0x05
    case objectId = 0x07
    case boolean = 0x08
    case utcDateTime = 0x09
    case nullValue = 0x0A
    case regex = 0x0B
    case javascriptCode = 0x0D
    case javascriptCodeWithScope = 0x0F
    case int32 = 0x10
    case timestamp = 0x11
    case int64 = 0x12
    case minKey = 0xFF
    case maxKey = 0x7F
}

/// `Document` is a collection type that uses a BSON document as storage.
/// As such, it can be stored in a file or instantiated from BSON data.
{'fraction_non_alphanumeric': 0.08292175721341305, 'fraction_numerical': 0.03275279438523525, 'mean_word_length': 1.661134163208852, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1421167', 'n_tokens_mistral': 1163, 'n_tokens_neox': 1058, 'n_words': 322}