<commit_before>//
//  BSFS.swift
//  BartlebyKit
//
//  Created by <NAME> on 02/11/2016.
//
//
import Foundation


/// The BSFS is set per document.
/// File level operations are done on GCD global utility queue.
public final class BSFS:TriggerHook{

    // MARK: -

    // Document
    fileprivate var _document:BartlebyDocument

    /// The File manager used to perform all the BSFS operation on GCD global utility queue.
    /// Note that we also use specific FileHandle at chunk level
    fileprivate let _fileManager:FileManager=FileManager()

    /// Chunk level operations
    fileprivate let _chunker:Chunker

    // The box Delegate
    fileprivate var _boxDelegate:BoxDelegate?

    // The current accessors key:node.UID, value:Array of Accessors
    fileprivate var _accessors=[String:[NodeAccessor]]()


    /// Store the blocks UIDs that need to be uploaded
    fileprivate var _toBeUploadedBlocksUIDS=[String]()

    // Store the blocks UIDs that need to be downloaded
    fileprivate var _toBeDownloadedBlocksUIDS=[String]()

    /// The downloads operations in progrees
    fileprivate var _downloadsInProgress=[DownloadBlock]()

    /// The uploads operations in progrees
    fileprivate var _uploadsInProgress=[UploadBlock]()

    /// Max Simultaneous operations ( per operation type and per bsfs instance. )
    fileprivate var _maxSimultaneousOperations=1

    // MARK: - initialization

    /// Each document has it own BSFS
    ///
    /// - Parameter document: the document instance
    required public init(in document:BartlebyDocument){
        self._document=document
        self._chunker=Chunker(fileManager: FileManager.default,
                              mode:.digestAndProcessing,
                              embeddedIn:document)
        self._boxDelegate=document
    }


    // MARK: - Persistency

    /// Serialize the local state of the BSFS
    ///
    /// - Returns: the serialized data
    public func saveState()->Data{
        let state=["toBeUploaded":self._toBeUploadedBlocksUIDS,
                   "toBeDownloaded":self._toBeDownloadedBlocksUIDS]
        if let data = try? JSONSerialization.data(withJSONObject: state){
            return data
        }
        return Data()
    }

    /// Restore the state
    ///
    /// - Parameter data: from the serialized state data
    public func restoreStateFrom(data:Data)throws->(){
        if let state = try? JSONSerialization.jsonObject(with: data) as? [String:[String]]{
            self._toBeDownloadedBlocksUIDS=state?["toBeDownloaded"] ?? [String]()
            self._toBeUploadedBlocksUIDS=state?["toBeUploaded"] ?? [String]()
        }
    }


    // MARK: - Paths

    /// The BSFS base folder path
    /// ---
    /// baseFolder/
    ///     - boxes/<boxUID>/[files]
    ///     - downloads/[files] tmp download files
    public var baseFolderPath:String{
        if self._document.metadata.appGroup != ""{
            if let url=self._fileManager.containerURL(forSecurityApplicationGroupIdentifier: self._document.metadata.appGroup){
                return url.path+"/\(_document.UID)"
            }
        }
        return Bartleby.getSearchPath(.documentDirectory)!+"/\(_document.UID)"

    }

    /// Downloads folder
    public var downloadFolderPath:String{
        return baseFolderPath+"/downloads"
    }

    /// Boxes folder
    public var boxesFolderPath:String{
        return baseFolderPath+"/boxes"
    }


    //MARK:  - Box Level

    /// Mounts the current local box == Assemble all its assemblable nodes
    /// There is no guarantee that the box is not fully up to date
    ///
    /// - Parameters:
    ///   - boxUID: the Box UID
    ///   - progressed: a closure  to relay the Progression State
    ///   - completed: a closure called on completion with Completion State.
    ///                the box UID is stored in the completion.getResultExternalReference()
    public func mount( boxUID:String,
                       progressed:@escaping (Progression)->(),
                       completed:@escaping (Completion)->()){
        do {

            let box = try Bartleby.registredObjectByUID(boxUID) as Box
            self._document.send(BoxStates.isMounting(box: box))
            if box.assemblyInProgress || box.isMounted {
                throw BSFSError.attemptToMountBoxMultipleTime(boxUID: boxUID)
            }

            var concernedNodes=[Node]()

            try? self._fileManager.createDirectory(atPath: box.nodesFolderPath, withIntermediateDirectories: true, attributes: nil)

            // Let's try to assemble as much nodes as we can.
            let nodes=box.nodes
            for node in nodes{
                let isNotAssembled = !self.isAssembled(node)
                let isAssemblable = node.isAssemblable
                if node.isAssemblable && isNotAssembled{
                    concernedNodes.append(node)
                }
            }

            box.quietChanges{
                box.assemblyProgression.totalTaskCount=concernedNodes.count
                box.assemblyProgression.currentTaskIndex=0
                box.assemblyProgression.currentPercentProgress=0
            }

            // We want to assemble the node sequentially.
            // So we will use a recursive pop method
            func __popNode(){
                if let node=concernedNodes.popLast(){
                    node.assemblyInProgress=true
                    do{
                        try self._assemble(node: node, progressed: { (progression) in
                            progressed(progression)
                        }, completed: { (completion) in
                            node.assemblyInProgress=false
                            box.assemblyProgression.currentTaskIndex += 1
                            box.assemblyProgression.currentPercentProgress=Double(box.assemblyProgression.currentTaskIndex)*Double(100)/Double(box.assemblyProgression.totalTaskCount)
                            __popNode()
                        })
                    }catch{
                        self._document.log("\(error)", file: #file, function: #function, line: #line, category: Default.LOG_DEFAULT, decorative: false)
                        completed(Completion.failureStateFromError(error))
                    }

                }else{
                    box.assemblyInProgress=false
                    box.isMounted=true
                    let completionState=Completion.successState()
                    completionState.setExternalReferenceResult(from:box)
                    completed(completionState)
                    self._document.send(BoxStates.hasBeenMounted(box: box))
                }
            }
            // Call the first pop.
            __popNode()

        } catch {
            completed(Completion.failureStateFromError(error))
            self._document.send(BoxStates.mountingHasFailed(boxUID:boxUID,message: error.localizedDescription))
        }
    }


    public func unMountAllBoxes(){
        self._document.boxes.forEach { (box) in
            self.unMount(box: box)
        }
        try? self._fileManager.removeItem(atPath: self.boxesFolderPath)
    }

    /// Un mounts the Box == deletes all the assembled files
    ///
    /// - Parameters:
    ///   - boxUID: the Box UID
    ///   - completed: a closure called on completion with Completion State.
    public func unMount( boxUID:String,
                         completed:@escaping (Completion)->()){
        do {
            let box = try Bartleby.registredObjectByUID(boxUID) as Box
            self.unMount(box: box)
            completed(Completion.successState())
        }catch{
            completed(Completion.failureStateFromError(error))
        }
    }


    public func unMount(box:Box)->(){
        for node in box.nodes{
            if let accessors=self._accessors[node.UID]{
                for accessor in accessors{
                    accessor.willBecomeUnusable(node: node)
                }
            }
            let assembledPath=self.assemblyPath(for:node)
            do{
                try self._fileManager.removeItem(atPath: assembledPath)
            }catch{
            }
        }
        do{
            try self._fileManager.removeItem(atPath: box.nodesFolderPath)
        }catch{
        }
        box.isMounted=false
        box.assemblyInProgress=false
    }

    ///
    /// - Parameter node: the node
    /// - Returns: the assembled path (created if there no
    public func assemblyPath(for node:Node)->String{
        if let box=node.box{
            return self.assemblyPath(for: box)+node.relativePath
        }

        return Default.NO_PATH
    }


    public func assemblyPath(for box:Box)->String{
        return  box.nodesFolderPath
    }


    /// Return is the node file has been assembled
    ///
    /// - Parameter node: the node
    /// - Returns: true if the file is available and the node not marked assemblyInProgress
    public func isAssembled(_ node:Node)->Bool{
        if node.assemblyInProgress {
            // Return false if the assembly is in progress
            return false
        }
        let group=AsyncGroup()
        var isAssembled=false
        group.utility{
            let path=self.assemblyPath(for: node)
            isAssembled=self._fileManager.fileExists(atPath: path)
            if isAssembled{
                if let attributes = try? self._fileManager.attributesOfItem(atPath: path){
                    if let size:Int=attributes[FileAttributeKey.size] as? Int{
                        if size != node.size{
                            self._document.log("Divergent size node Size:\(node.size) fs.size: \(size) ", file: #file, function: #function, line: #line, category: Default.LOG_DEFAULT, decorative: false)
                            isAssembled=false
                        }
                    }
                }
            }
        }
        group.wait()
        return isAssembled

    }


    /// Creates a file from the node blocks.
    /// IMPORTANT: this method requires a response from the BoxDelegate
    ///
    /// - Parameters:
    ///   - node: the node
    ///   - progressed: a closure  to relay the Progression State
    ///   - completed: a closure called on completion with Completion State.
    ///                the node UID is stored in the completion.getResultExternalReference()
    internal func _assemble(node:Node,
                            progressed:@escaping (Progression)->(),
                            completed:@escaping (Completion)->()) throws->(){


        do {
            if node.isAssemblable == false{
                throw BSFSError.nodeIsNotAssemblable
            }
            if let delegate = self._boxDelegate{
                delegate.nodeIsReady(node: node, proceed: {

                    let path=self.assemblyPath(for: node)

                    let blocks=node.blocks.sorted(by: { (rblock, lblock) -> Bool in
                        return rblock.rank < lblock.rank
                    })

                    if node.nature == .file || node.nature == .flock {

                        var blockPaths=[String]()
                        for block in blocks{
                            // The blocks are embedded in a document
                            // So we use the relative path
                            blockPaths.append(block.blockRelativePath())
                        }
                        self._chunker.joinChunksToFile(from: blockPaths,
                                                       to: path,
                                                       decompress: node.compressedBlocks,
                                                       decrypt: node.cryptedBlocks,
                                                       externalId:node.UID,
                                                       progression: { (progression) in
                                                        progressed(progression)
                        }, success: { path in

                            if node.nature == .flock{
                                //TODO

                                // FLOCK path == path

                                let completionState=Completion.successState()
                                completionState.setExternalReferenceResult(from:node)
                                completed(completionState)

                            }else{
                                // The file has been assembled
                                let completionState=Completion.successState()
                                completionState.setExternalReferenceResult(from:node)
                                completed(completionState)
                            }
                        }, failure: { (message) in
                            let completion=Completion()
                            completion.message=message
                            completion.success=false
                            completion.externalIdentifier=node.UID
                            completed(completion)
                        })
                    }else if node.nature == .alias{
                        Async.utility{
                            do{
                                if let rNodeUID=node.referentNodeUID{
                                    if let rNode = try? Bartleby.registredObjectByUID(rNodeUID) as Node{
                                        let destination=self.assemblyPath(for: rNode)
                                        try self._fileManager.createSymbolicLink(atPath: path, withDestinationPath:destination)
                                        let completionState=Completion.successState()
                                        completionState.setExternalReferenceResult(from:node)
                                        completed(completionState)
                                    }else{
                                        completed(Completion.failureState("Unable to find Alias referent node", statusCode:.expectation_Failed))
                                    }
                                }else{
                                    completed(Completion.failureState("Unable to find Alias destination", statusCode:.expectation_Failed))

                                }
                            }catch{
                                completed(Completion.failureStateFromError(error))
                            }
                        }

                    }else if node.nature == .folder{
                        Async.utility{
                            do{
                                try self._fileManager.createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil)
                                let completionState=Completion.successState()
                                completionState.setExternalReferenceResult(from:node)
                                completed(completionState)
                            }catch{
                                completed(Completion.failureStateFromError(error))
                            }
                        }

                    }
                })
            }else{
                throw BSFSError.boxDelegateIsNotAvailable
            }

        } catch{
            completed(Completion.failureStateFromError(error))
        }

    }


    //MARK:  - File Level


    /// Adds a file or all the file from o folder into the box
    ///     Flocks may be supported soon
    ///
    ///     + generate the blocks in background.
    ///     + adds the node(s)
    ///     + the first node UID is stored in the Completion (use completion.getResultExternalReference())
    ///
    /// - Parameters:
    ///   - FileReference: the file reference (Nature == file or flock)
    ///   - box: the box
    ///   - relativePath: the relative Path of the Node in the box
    ///   - deleteOriginal: should we delete the original?
    ///   - progressed: a closure  to relay the Progression State
    ///   - completed: a closure called on completion with Completion State
    ///                the node UID is stored in the completion.getResultExternalReference()
    public func add( reference:FileReference,
                     in box:Box,
                     to relativePath:String,
                     deleteOriginal:Bool=false,
                     progressed:@escaping (Progression)->(),
                     completed:@escaping (Completion)->()){

        if self._fileManager.fileExists(atPath: reference.absolutePath){

            var firstNode:Node?
            func __chunksHaveBeenCreated(chunks:[Chunk]){

                // Successful operation
                // Let's Upsert the distant models.
                // AND  the local nodes

                let groupedChunks=Chunk.groupByNodePath(chunks: chunks)

                for (nodeRelativePath,groupOfChunks) in groupedChunks{
                    // Create the new node.
                    // And add its blocks
                    let node=self._document.newManagedModel() as Node
                    if firstNode==nil{
                        firstNode=node
                    }
                    node.quietChanges{
                        node.nature=reference.nodeNature.forNode
                        node.relativePath=relativePath///
                        node.priority=reference.priority
                        // Set up the node relative path
                        node.relativePath=nodeRelativePath

                        var cumulatedDigests=""
                        var cumulatedSize=0

                        box.quietChanges {
                            box.declaresOwnership(of: node)
                        }

                        // Let's add the blocks
                        for chunk in groupOfChunks{
                            let block=self._document.newManagedModel() as Block
                            block.quietChanges{
                                block.rank=chunk.rank
                                block.digest=chunk.sha1
                                block.startsAt=chunk.startsAt
                                block.size=chunk.originalSize
                                block.priority=reference.priority

                            }
                            cumulatedSize += chunk.originalSize
                            cumulatedDigests += chunk.sha1
                            node.addBlock(block)
                            self._toBeUploadedBlocksUIDS.append(block.UID)
                        }
                        // Store the digest of the cumulated digests.
                        node.digest=cumulatedDigests.sha1
                        // And the node original size
                        node.size=cumulatedSize
                    }

                }


                // Delete the original
                if deleteOriginal{
                    // We consider deletion as non mandatory.
                    // So we produce only a log.
                    do {
                        try self._fileManager.removeItem(atPath: reference.absolutePath)
                    } catch  {
                        self._document.log("Deletion has failed. Path:\( reference.absolutePath)", file: #file, function: #function, line: #line, category: Default.LOG_DEFAULT, decorative: false)
                    }
                }

                let finalState=Completion.successState()
                if let node=firstNode{
                    finalState.setExternalReferenceResult(from:node)
                }
                completed(finalState)

                // Call the centralized upload mechanism
                self._uploadNext()

            }
            Async.utility{
                if  reference.nodeNature == .file{
                    var isDirectory:ObjCBool=false
                    if self._fileManager.fileExists(atPath: reference.absolutePath, isDirectory: &isDirectory){
                        if isDirectory.boolValue{
                            self._chunker.breakFolderIntoChunk(filesIn: reference.absolutePath,
                                                               chunksFolderPath: box.nodesFolderPath,
                                                               progression: { (progression) in
                                                                progressed(progression)<commit_msg>Enforced Thread Safety on BSFS<commit_after>//
//  BSFS.swift
//  BartlebyKit
//
//  Created by <NAME> on 02/11/2016.
//
//
import Foundation


/// The BSFS is set per document.
/// File level operations are done on GCD global utility queue.
public final class BSFS:TriggerHook{

    // MARK: -

    // Document
    fileprivate var _document:BartlebyDocument

    /// The File manager used to perform all the BSFS operation on GCD global utility queue.
    /// Note that we also use specific FileHandle at chunk level
    fileprivate let _fileManager:FileManager=FileManager()

    /// Chunk level operations
    fileprivate let _chunker:Chunker

    // The box Delegate
    fileprivate var _boxDelegate:BoxDelegate?

    // The current accessors key:node.UID, value:Array of Accessors
    fileprivate var _accessors=[String:[NodeAccessor]]()


    /// Store the blocks UIDs that need to be uploaded
    fileprivate var _toBeUploadedBlocksUIDS=[String]()

    // Store the blocks UIDs that need to be downloaded
    fileprivate var _toBeDownloadedBlocksUIDS=[String]()

    /// The downloads operations in progrees
    fileprivate var _downloadsInProgress=[DownloadBlock]()

    /// The uploads operations in progrees
    fileprivate var _uploadsInProgress=[UploadBlock]()

    /// Max Simultaneous operations ( per operation type and per bsfs instance. )
    fileprivate var _maxSimultaneousOperations=1

    // MARK: - initialization

    /// Each document has it own BSFS
    ///
    /// - Parameter document: the document instance
    required public init(in document:BartlebyDocument){
        self._document=document
        self._chunker=Chunker(fileManager: FileManager.default,
                              mode:.digestAndProcessing,
                              embeddedIn:document)
        self._boxDelegate=document
    }


    // MARK: - Persistency

    /// Serialize the local state of the BSFS
    ///
    /// - Returns: the serialized data
    public func saveState()->Data{
        let state=["toBeUploaded":self._toBeUploadedBlocksUIDS,
                   "toBeDownloaded":self._toBeDownloadedBlocksUIDS]
        if let data = try? JSONSerialization.data(withJSONObject: state){
            return data
        }
        return Data()
    }

    /// Restore the state
    ///
    /// - Parameter data: from the serialized state data
    public func restoreStateFrom(data:Data)throws->(){
        if let state = try? JSONSerialization.jsonObject(with: data) as? [String:[String]]{
            self._toBeDownloadedBlocksUIDS=state?["toBeDownloaded"] ?? [String]()
            self._toBeUploadedBlocksUIDS=state?["toBeUploaded"] ?? [String]()
        }
    }


    // MARK: - Paths

    /// The BSFS base folder path
    /// ---
    /// baseFolder/
    ///     - boxes/<boxUID>/[files]
    ///     - downloads/[files] tmp download files
    public var baseFolderPath:String{
        if self._document.metadata.appGroup != ""{
            if let url=self._fileManager.containerURL(forSecurityApplicationGroupIdentifier: self._document.metadata.appGroup){
                return url.path+"/\(_document.UID)"
            }
        }
        return Bartleby.getSearchPath(.documentDirectory)!+"/\(_document.UID)"

    }

    /// Downloads folder
    public var downloadFolderPath:String{
        return baseFolderPath+"/downloads"
    }

    /// Boxes folder
    public var boxesFolderPath:String{
        return baseFolderPath+"/boxes"
    }


    //MARK:  - Box Level

    /// Mounts the current local box == Assemble all its assemblable nodes
    /// There is no guarantee that the box is not fully up to date
    ///
    /// - Parameters:
    ///   - boxUID: the Box UID
    ///   - progressed: a closure  to relay the Progression State
    ///   - completed: a closure called on completion with Completion State.
    ///                the box UID is stored in the completion.getResultExternalReference()
    public func mount( boxUID:String,
                       progressed:@escaping (Progression)->(),
                       completed:@escaping (Completion)->()){
        do {

            let box = try Bartleby.registredObjectByUID(boxUID) as Box
            self._document.send(BoxStates.isMounting(box: box))
            if box.assemblyInProgress || box.isMounted {
                throw BSFSError.attemptToMountBoxMultipleTime(boxUID: boxUID)
            }

            var concernedNodes=[Node]()

            try? self._fileManager.createDirectory(atPath: box.nodesFolderPath, withIntermediateDirectories: true, attributes: nil)

            // Let's try to assemble as much nodes as we can.
            let nodes=box.nodes
            for node in nodes{
                let isNotAssembled = !self.isAssembled(node)
                let isAssemblable = node.isAssemblable
                if node.isAssemblable && isNotAssembled{
                    concernedNodes.append(node)
                }
            }

            box.quietChanges{
                box.assemblyProgression.totalTaskCount=concernedNodes.count
                box.assemblyProgression.currentTaskIndex=0
                box.assemblyProgression.currentPercentProgress=0
            }

            // We want to assemble the node sequentially.
            // So we will use a recursive pop method
            func __popNode(){
                if let node=concernedNodes.popLast(){
                    node.assemblyInProgress=true
                    do{
                        try self._assemble(node: node, progressed: { (progression) in
                            progressed(progression)
                        }, completed: { (completion) in
                            node.assemblyInProgress=false
                            box.assemblyProgression.currentTaskIndex += 1
                            box.assemblyProgression.currentPercentProgress=Double(box.assemblyProgression.currentTaskIndex)*Double(100)/Double(box.assemblyProgression.totalTaskCount)
                            __popNode()
                        })
                    }catch{
                        self._document.log("\(error)", file: #file, function: #function, line: #line, category: Default.LOG_DEFAULT, decorative: false)
                        completed(Completion.failureStateFromError(error))
                    }

                }else{
                    box.assemblyInProgress=false
                    box.isMounted=true
                    let completionState=Completion.successState()
                    completionState.setExternalReferenceResult(from:box)
                    completed(completionState)
                    self._document.send(BoxStates.hasBeenMounted(box: box))
                }
            }
            // Call the first pop.
            __popNode()

        } catch {
            completed(Completion.failureStateFromError(error))
            self._document.send(BoxStates.mountingHasFailed(boxUID:boxUID,message: error.localizedDescription))
        }
    }


    public func unMountAllBoxes(){
        self._document.boxes.forEach { (box) in
            self.unMount(box: box)
        }
        try? self._fileManager.removeItem(atPath: self.boxesFolderPath)
    }

    /// Un mounts the Box == deletes all the assembled files
    ///
    /// - Parameters:
    ///   - boxUID: the Box UID
    ///   - completed: a closure called on completion with Completion State.
    public func unMount( boxUID:String,
                         completed:@escaping (Completion)->()){
        do {
            let box = try Bartleby.registredObjectByUID(boxUID) as Box
            self.unMount(box: box)
            completed(Completion.successState())
        }catch{
            completed(Completion.failureStateFromError(error))
        }
    }


    public func unMount(box:Box)->(){
        for node in box.nodes{
            if let accessors=self._accessors[node.UID]{
                for accessor in accessors{
                    accessor.willBecomeUnusable(node: node)
                }
            }
            let assembledPath=self.assemblyPath(for:node)
            do{
                try self._fileManager.removeItem(atPath: assembledPath)
            }catch{
            }
        }
        do{
            try self._fileManager.removeItem(atPath: box.nodesFolderPath)
        }catch{
        }
        box.isMounted=false
        box.assemblyInProgress=false
    }

    ///
    /// - Parameter node: the node
    /// - Returns: the assembled path (created if there no
    public func assemblyPath(for node:Node)->String{
        if let box=node.box{
            return self.assemblyPath(for: box)+node.relativePath
        }

        return Default.NO_PATH
    }


    public func assemblyPath(for box:Box)->String{
        return  box.nodesFolderPath
    }


    /// Return is the node file has been assembled
    ///
    /// - Parameter node: the node
    /// - Returns: true if the file is available and the node not marked assemblyInProgress
    public func isAssembled(_ node:Node)->Bool{
        if node.assemblyInProgress {
            // Return false if the assembly is in progress
            return false
        }
        let group=AsyncGroup()
        var isAssembled=false
        group.utility{
            let path=self.assemblyPath(for: node)
            isAssembled=self._fileManager.fileExists(atPath: path)
            if isAssembled{
                if let attributes = try? self._fileManager.attributesOfItem(atPath: path){
                    if let size:Int=attributes[FileAttributeKey.size] as? Int{
                        if size != node.size{
                            self._document.log("Divergent size node Size:\(node.size) fs.size: \(size) ", file: #file, function: #function, line: #line, category: Default.LOG_DEFAULT, decorative: false)
                            isAssembled=false
                        }
                    }
                }
            }
        }
        group.wait()
        return isAssembled

    }


    /// Creates a file from the node blocks.
    /// IMPORTANT: this method requires a response from the BoxDelegate
    ///
    /// - Parameters:
    ///   - node: the node
    ///   - progressed: a closure  to relay the Progression State
    ///   - completed: a closure called on completion with Completion State.
    ///                the node UID is stored in the completion.getResultExternalReference()
    internal func _assemble(node:Node,
                            progressed:@escaping (Progression)->(),
                            completed:@escaping (Completion)->()) throws->(){


        do {
            if node.isAssemblable == false{
                throw BSFSError.nodeIsNotAssemblable
            }
            if let delegate = self._boxDelegate{
                delegate.nodeIsReady(node: node, proceed: {

                    let path=self.assemblyPath(for: node)

                    let blocks=node.blocks.sorted(by: { (rblock, lblock) -> Bool in
                        return rblock.rank < lblock.rank
                    })

                    if node.nature == .file || node.nature == .flock {

                        var blockPaths=[String]()
                        for block in blocks{
                            // The blocks are embedded in a document
                            // So we use the relative path
                            blockPaths.append(block.blockRelativePath())
                        }
                        self._chunker.joinChunksToFile(from: blockPaths,
                                                       to: path,
                                                       decompress: node.compressedBlocks,
                                                       decrypt: node.cryptedBlocks,
                                                       externalId:node.UID,
                                                       progression: { (progression) in
                                                        progressed(progression)
                        }, success: { path in

                            if node.nature == .flock{
                                //TODO

                                // FLOCK path == path

                                let completionState=Completion.successState()
                                completionState.setExternalReferenceResult(from:node)
                                completed(completionState)

                            }else{
                                // The file has been assembled
                                let completionState=Completion.successState()
                                completionState.setExternalReferenceResult(from:node)
                                completed(completionState)
                            }
                        }, failure: { (message) in
                            let completion=Completion()
                            completion.message=message
                            completion.success=false
                            completion.externalIdentifier=node.UID
                            completed(completion)
                        })
                    }else if node.nature == .alias{
                        Async.utility{
                            do{
                                if let rNodeUID=node.referentNodeUID{
                                    if let rNode = try? Bartleby.registredObjectByUID(rNodeUID) as Node{
                                        let destination=self.assemblyPath(for: rNode)
                                        try self._fileManager.createSymbolicLink(atPath: path, withDestinationPath:destination)
                                        let completionState=Completion.successState()
                                        completionState.setExternalReferenceResult(from:node)
                                        completed(completionState)
                                    }else{
                                        completed(Completion.failureState("Unable to find Alias referent node", statusCode:.expectation_Failed))
                                    }
                                }else{
                                    completed(Completion.failureState("Unable to find Alias destination", statusCode:.expectation_Failed))

                                }
                            }catch{
                                completed(Completion.failureStateFromError(error))
                            }
                        }

                    }else if node.nature == .folder{
                        Async.utility{
                            do{
                                try self._fileManager.createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil)
                                let completionState=Completion.successState()
                                completionState.setExternalReferenceResult(from:node)
                                completed(completionState)
                            }catch{
                                completed(Completion.failureStateFromError(error))
                            }
                        }

                    }
                })
            }else{
                throw BSFSError.boxDelegateIsNotAvailable
            }

        } catch{
            completed(Completion.failureStateFromError(error))
        }

    }


    //MARK:  - File Level


    /// Adds a file or all the file from o folder into the box
    ///     Flocks may be supported soon
    ///
    ///     + generate the blocks in background.
    ///     + adds the node(s)
    ///     + the first node UID is stored in the Completion (use completion.getResultExternalReference())
    ///
    /// - Parameters:
    ///   - FileReference: the file reference (Nature == file or flock)
    ///   - box: the box
    ///   - relativePath: the relative Path of the Node in the box
    ///   - deleteOriginal: should we delete the original?
    ///   - progressed: a closure  to relay the Progression State
    ///   - completed: a closure called on completion with Completion State
    ///                the node UID is stored in the completion.getResultExternalReference()
    public func add( reference:FileReference,
                     in box:Box,
                     to relativePath:String,
                     deleteOriginal:Bool=false,
                     progressed:@escaping (Progression)->(),
                     completed:@escaping (Completion)->()){

        if self._fileManager.fileExists(atPath: reference.absolutePath){

            var firstNode:Node?
            func __chunksHaveBeenCreated(chunks:[Chunk]){

                Bartleby.syncOnMain{
                    // Successful operation
                    // Let's Upsert the distant models.
                    // AND  the local nodes

                    let groupedChunks=Chunk.groupByNodePath(chunks: chunks)

                    for (nodeRelativePath,groupOfChunks) in groupedChunks{
                        // Create the new node.
                        // And add its blocks
                        let node=self._document.newManagedModel() as Node
                        if firstNode==nil{
                            firstNode=node
                        }
                        node.quietChanges{
                            node.nature=reference.nodeNature.forNode
                            node.relativePath=relativePath///
                            node.priority=reference.priority
                            // Set up the node relative path
                            node.relativePath=nodeRelativePath

                            var cumulatedDigests=""
                            var cumulatedSize=0

                            box.quietChanges {
                                box.declaresOwnership(of: node)
                            }

                            // Let's add the blocks
                            for chunk in groupOfChunks{
                                let block=self._document.newManagedModel() as Block
                                block.quietChanges{
                                    block.rank=chunk.rank
                                    block.digest=chunk.sha1
                                    block.startsAt=chunk.startsAt
                                    block.size=chunk.originalSize
                                    block.priority=reference.priority

                                }
                                cumulatedSize += chunk.originalSize
                                cumulatedDigests += chunk.sha1
                                node.addBlock(block)
                                self._toBeUploadedBlocksUIDS.append(block.UID)
                            }
                            // Store the digest of the cumulated digests.
                            node.digest=cumulatedDigests.sha1
                            // And the node original size
                            node.size=cumulatedSize
                        }
                    }
                }

                Async.utility{
                    // Delete the original
                    if deleteOriginal{
                        // We consider deletion as non mandatory.
                        // So we produce only a log.
                        do {
                            try self._fileManager.removeItem(atPath: reference.absolutePath)
                        } catch  {
                            self._document.log("Deletion has failed. Path:\( reference.absolutePath)", file: #file, function: #function, line: #line, category: Default.LOG_DEFAULT, decorative: false)
                        }
                    }

                    let finalState=Completion.successState()
                    if let node=firstNode{
                        finalState.setExternalReferenceResult(from:node)
                    }
                    Bartleby.syncOnMain{
                        completed(finalState)
                    }

                    // Call the centralized upload mechanism
                    self._uploadNext()
                }

            }

            Async.utility{
                if  reference.nodeNature == .file{
                    var isDirectory:ObjCBool=false
                    if self._fileManager.fileExists(atPath: reference.absolutePath, isDirectory: &isDirectory){
                        if isDirectory.boolValue{
                            self._chunker.breakFolderIntoChunk(filesIn: reference.absolutePath,
                                                               chunksFolderPath: box.nodesFolderPath,
                                                               progression: { (progression) in
                                                                progressed(progression)
{'fraction_non_alphanumeric': 0.07711328210225207, 'fraction_numerical': 0.0009211897893384403, 'mean_word_length': 1.295987087438081, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 117, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '214226', 'n_tokens_mistral': 9606, 'n_tokens_neox': 8962, 'n_words': 2266}