<commit_before>//
// This source file is part of the MongoKitten open source project
//
// Copyright (c) 2016 - 2017 OpenKitten and the MongoKitten project authors
// Licensed under MIT
//
// See https://github.com/OpenKitten/MongoKitten/blob/mongokitten31/LICENSE.md for license information
// See https://github.com/OpenKitten/MongoKitten/blob/mongokitten31/CONTRIBUTORS.md for the list of MongoKitten project authors
//

import BSON
import Foundation

#if (os(macOS) || os(iOS)) && !OPENSSL
    import CryptoSwift
#else
    import KittenCTLS
#endif

/// A GridFS instance similar to a collection
///
/// Conforms to the GridFS standard as specified here: https://docs.mongodb.org/manual/core/gridfs/
public final class GridFS {
    /// The bucket for file data
    public let files: Collection
    
    /// The bucket for file data chunks
    public let chunks: Collection
    
    /// The GridFS bucket name
    public let name: String
    
    /// Drops the GridFS bucket's collections
    public func drop() throws {
        try self.files.drop()
        try self.chunks.drop()
    }
    
    /// Initializes a GridFS `Collection` (bucket) in a given database
    ///
    /// - parameter in: In which database does this GridFS bucket reside
    /// - parameter named: The optional name of this GridFS bucket (by default "fs")
    ///
    /// - throws: When unable to send the request/receive the response, the authenticated user doesn't have sufficient permissions or an error occurred or when it can't create it's indexes
    public init(in database: Database, named bucketName: String = "fs") throws {
        files = database["\(bucketName).files"]
        chunks = database["\(bucketName).chunks"]
        name = bucketName
        
        // Make indexes
        try chunks.createIndex(named: "chunksindex", withParameters: .sortedCompound(fields: [("files_id", .ascending), ("n", .ascending)]), .buildInBackground, .unique)
        
        try files.createIndex(named: "filename", withParameters: .sortedCompound(fields: [("uploadDate", .ascending), ("filesindex", .ascending)]), .buildInBackground)
    }
    
    /// Finds the first file matching this ObjectID
    ///
    /// - parameter byID: The hash to look for
    ///
    /// - throws: When unable to send the request/receive the response, the authenticated user doesn't have sufficient permissions or an error occurred
    ///
    /// - returns: The resulting file
    public func findOne(byID id: ObjectId) throws -> File? {
        return try self.find("_id" == id).makeIterator().next()
    }
    
    /// Finds using a matching filter
    ///
    /// - parameter filter: The filter to use
    ///
    /// - throws: When unable to send the request/receive the response, the authenticated user doesn't have sufficient permissions or an error occurred
    ///
    /// - returns: A cursor pointing to all resulting files
    public func find(_ filter: Query? = nil) throws -> Cursor<File> {
        return try files.find(filter).flatMap { doc -> File? in
            return File(document: doc, chunksCollection: self.chunks, filesCollection: self.files)
        }.cursor
    }
    
    /// Removes a file by it's identifier
    public func remove(byId identifier: ObjectId) throws {
        try files.remove("_id" == identifier)
        try chunks.remove("files_id" == identifier)
    }
    
    /// Stores the data in GridFS
    ///
    /// - parameter data: The data to store
    /// - parameter named: The optional filename to use for this data
    /// - parameter withType: The optional MIME type to use for this data
    /// - parameter usingMetadata: The optional metadata to store with this file
    /// - parameter inChunksOf: The amount of bytes to put in one chunk
    ///
    /// TODO: Accept data streams
    public func store(data binary: Bytes, named filename: String? = nil, withType contentType: String? = nil, usingMetadata metadata: BSON.Primitive? = nil, inChunksOf chunkSize: Int = 255_000) throws -> ObjectId {
        guard chunkSize < 15_000_000 else {
            throw MongoError.invalidChunkSize(chunkSize: chunkSize)
        }
        
        var data = binary
        let id = ObjectId()
        let dataSize = data.count
        
        #if (os(macOS) || os(iOS)) && !OPENSSL
            var context = MD5()
            let lastChunkSize = (data.count % 255_000)
            let endStart = data.count - lastChunkSize
            let lastChunk = Array(data[endStart..<data.count])
            data.removeLast(lastChunkSize)
        #else
            var context = MD5_CTX()
            guard MD5_Init(&context) == 1 else {
                throw MongoError.couldNotHashFile
            }
        #endif
        
        var insertData: Document = [
            "_id": id,
            "length": dataSize,
            "chunkSize": Int32(chunkSize),
            "uploadDate": Date(timeIntervalSinceNow: 0),
        ]
        
        if let filename = filename {
            insertData["filename"] = filename
        }
        
        if let contentType = contentType {
            insertData["contentType"] = contentType
        }
        
        if let metadata = metadata {
            insertData["metadata"] = metadata
        }
        
        var n: Int32 = 0
        
        do {
            while !data.isEmpty {
                let smallestMax = min(data.count, chunkSize)
                
                let chunk = Array(data[0..<smallestMax])
                
                #if (os(macOS) || os(iOS)) && !OPENSSL
                    _ = try context.update(withBytes: chunk)
                #else
                    guard MD5_Update(&context, chunk, chunk.count) == 1 else {
                        throw MongoError.couldNotHashFile
                    }
                #endif
                
                _ = try chunks.insert(["files_id": id,
                                       "n": n,
                                       "data": Binary(data: Array(chunk), withSubtype: .generic)])
                
                n += 1
                
                data.removeFirst(smallestMax)
            }
            
            #if (os(macOS) || os(iOS)) && !OPENSSL
                let digest = try context.update(withBytes: lastChunk, isLast: true)
                
                _ = try chunks.insert(["files_id": id,
                                       "n": n,
                                       "data": Binary(data: Array(lastChunk), withSubtype: .generic)])
            #else
                var digest = Bytes(repeating: 0, count: Int(MD5_DIGEST_LENGTH))
                
                guard MD5_Final(&digest, &context) == 1 else {
                    throw MongoError.couldNotHashFile
                }
            #endif
            
            insertData["md5"] = digest.toHexString()
            
            _ = try files.insert(insertData)
        } catch {
            try chunks.remove("files_id" == id)
            throw error
        }
        
        return id
    }
    
    /// Stores the data in GridFS
    /// - parameter data: The data to store
    /// - parameter named: The optional filename to use for this data
    /// - parameter withType: The optional MIME type to use for this data
    /// - parameter usingMetadata: The optional metadata to store with this file
    /// - parameter inChunksOf: The amount of bytes to put in one chunk
    public func store(data: Data, named filename: String? = nil, withType contentType: String? = nil, usingMetadata metadata: BSON.Primitive? = nil, inChunksOf chunkSize: Int = 255000) throws -> ObjectId {
        return try self.store(data: Array(data), named: filename, withType: contentType, usingMetadata: metadata, inChunksOf: chunkSize)
    }
    
    /// A file in GridFS
    public class File: Sequence {
        /// The ObjectID for this file<commit_msg>Made GridFS.File public<commit_after>//
// This source file is part of the MongoKitten open source project
//
// Copyright (c) 2016 - 2017 OpenKitten and the MongoKitten project authors
// Licensed under MIT
//
// See https://github.com/OpenKitten/MongoKitten/blob/mongokitten31/LICENSE.md for license information
// See https://github.com/OpenKitten/MongoKitten/blob/mongokitten31/CONTRIBUTORS.md for the list of MongoKitten project authors
//

import BSON
import Foundation

#if (os(macOS) || os(iOS)) && !OPENSSL
    import CryptoSwift
#else
    import KittenCTLS
#endif

/// A GridFS instance similar to a collection
///
/// Conforms to the GridFS standard as specified here: https://docs.mongodb.org/manual/core/gridfs/
public final class GridFS {
    /// The bucket for file data
    public let files: Collection
    
    /// The bucket for file data chunks
    public let chunks: Collection
    
    /// The GridFS bucket name
    public let name: String
    
    /// Drops the GridFS bucket's collections
    public func drop() throws {
        try self.files.drop()
        try self.chunks.drop()
    }
    
    /// Initializes a GridFS `Collection` (bucket) in a given database
    ///
    /// - parameter in: In which database does this GridFS bucket reside
    /// - parameter named: The optional name of this GridFS bucket (by default "fs")
    ///
    /// - throws: When unable to send the request/receive the response, the authenticated user doesn't have sufficient permissions or an error occurred or when it can't create it's indexes
    public init(in database: Database, named bucketName: String = "fs") throws {
        files = database["\(bucketName).files"]
        chunks = database["\(bucketName).chunks"]
        name = bucketName
        
        // Make indexes
        try chunks.createIndex(named: "chunksindex", withParameters: .sortedCompound(fields: [("files_id", .ascending), ("n", .ascending)]), .buildInBackground, .unique)
        
        try files.createIndex(named: "filename", withParameters: .sortedCompound(fields: [("uploadDate", .ascending), ("filesindex", .ascending)]), .buildInBackground)
    }
    
    /// Finds the first file matching this ObjectID
    ///
    /// - parameter byID: The hash to look for
    ///
    /// - throws: When unable to send the request/receive the response, the authenticated user doesn't have sufficient permissions or an error occurred
    ///
    /// - returns: The resulting file
    public func findOne(byID id: ObjectId) throws -> File? {
        return try self.find("_id" == id).makeIterator().next()
    }
    
    /// Finds using a matching filter
    ///
    /// - parameter filter: The filter to use
    ///
    /// - throws: When unable to send the request/receive the response, the authenticated user doesn't have sufficient permissions or an error occurred
    ///
    /// - returns: A cursor pointing to all resulting files
    public func find(_ filter: Query? = nil) throws -> Cursor<File> {
        return try files.find(filter).flatMap { doc -> File? in
            return File(document: doc, chunksCollection: self.chunks, filesCollection: self.files)
        }.cursor
    }
    
    /// Removes a file by it's identifier
    public func remove(byId identifier: ObjectId) throws {
        try files.remove("_id" == identifier)
        try chunks.remove("files_id" == identifier)
    }
    
    /// Stores the data in GridFS
    ///
    /// - parameter data: The data to store
    /// - parameter named: The optional filename to use for this data
    /// - parameter withType: The optional MIME type to use for this data
    /// - parameter usingMetadata: The optional metadata to store with this file
    /// - parameter inChunksOf: The amount of bytes to put in one chunk
    ///
    /// TODO: Accept data streams
    public func store(data binary: Bytes, named filename: String? = nil, withType contentType: String? = nil, usingMetadata metadata: BSON.Primitive? = nil, inChunksOf chunkSize: Int = 255_000) throws -> ObjectId {
        guard chunkSize < 15_000_000 else {
            throw MongoError.invalidChunkSize(chunkSize: chunkSize)
        }
        
        var data = binary
        let id = ObjectId()
        let dataSize = data.count
        
        #if (os(macOS) || os(iOS)) && !OPENSSL
            var context = MD5()
            let lastChunkSize = (data.count % 255_000)
            let endStart = data.count - lastChunkSize
            let lastChunk = Array(data[endStart..<data.count])
            data.removeLast(lastChunkSize)
        #else
            var context = MD5_CTX()
            guard MD5_Init(&context) == 1 else {
                throw MongoError.couldNotHashFile
            }
        #endif
        
        var insertData: Document = [
            "_id": id,
            "length": dataSize,
            "chunkSize": Int32(chunkSize),
            "uploadDate": Date(timeIntervalSinceNow: 0),
        ]
        
        if let filename = filename {
            insertData["filename"] = filename
        }
        
        if let contentType = contentType {
            insertData["contentType"] = contentType
        }
        
        if let metadata = metadata {
            insertData["metadata"] = metadata
        }
        
        var n: Int32 = 0
        
        do {
            while !data.isEmpty {
                let smallestMax = min(data.count, chunkSize)
                
                let chunk = Array(data[0..<smallestMax])
                
                #if (os(macOS) || os(iOS)) && !OPENSSL
                    _ = try context.update(withBytes: chunk)
                #else
                    guard MD5_Update(&context, chunk, chunk.count) == 1 else {
                        throw MongoError.couldNotHashFile
                    }
                #endif
                
                _ = try chunks.insert(["files_id": id,
                                       "n": n,
                                       "data": Binary(data: Array(chunk), withSubtype: .generic)])
                
                n += 1
                
                data.removeFirst(smallestMax)
            }
            
            #if (os(macOS) || os(iOS)) && !OPENSSL
                let digest = try context.update(withBytes: lastChunk, isLast: true)
                
                _ = try chunks.insert(["files_id": id,
                                       "n": n,
                                       "data": Binary(data: Array(lastChunk), withSubtype: .generic)])
            #else
                var digest = Bytes(repeating: 0, count: Int(MD5_DIGEST_LENGTH))
                
                guard MD5_Final(&digest, &context) == 1 else {
                    throw MongoError.couldNotHashFile
                }
            #endif
            
            insertData["md5"] = digest.toHexString()
            
            _ = try files.insert(insertData)
        } catch {
            try chunks.remove("files_id" == id)
            throw error
        }
        
        return id
    }
    
    /// Stores the data in GridFS
    /// - parameter data: The data to store
    /// - parameter named: The optional filename to use for this data
    /// - parameter withType: The optional MIME type to use for this data
    /// - parameter usingMetadata: The optional metadata to store with this file
    /// - parameter inChunksOf: The amount of bytes to put in one chunk
    public func store(data: Data, named filename: String? = nil, withType contentType: String? = nil, usingMetadata metadata: BSON.Primitive? = nil, inChunksOf chunkSize: Int = 255000) throws -> ObjectId {
        return try self.store(data: Array(data), named: filename, withType: contentType, usingMetadata: metadata, inChunksOf: chunkSize)
    }
    
    /// A file in GridFS
    public final class File: Sequence {
        /// The ObjectID for this file
{'fraction_non_alphanumeric': 0.09881949733434882, 'fraction_numerical': 0.0072353389185072356, 'mean_word_length': 1.8426844668951832, 'pattern_counts': {'":': 20, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '660655', 'n_tokens_mistral': 4250, 'n_tokens_neox': 4044, 'n_words': 1372}