<commit_before>package be.cytomine.server

/*
 * Copyright (c) 2009-2019. Authors: see NOTICE file.
 *
 * Licensed under the GNU Lesser General Public License, Version 2.1 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      https://www.gnu.org/licenses/lgpl-2.1.txt
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import be.cytomine.client.CytomineException
import be.cytomine.client.models.*
import be.cytomine.exception.DeploymentException
import be.cytomine.exception.FormatException
import be.cytomine.exception.MiddlewareException
import be.cytomine.formats.*
import be.cytomine.formats.archive.ArchiveFormat
import be.cytomine.formats.supported.NativeFormat
import utils.FilesUtils

import java.nio.file.Paths

class UploadService {

    final static public String UNDERTERMINED_CONTENT = "undetermined"

    def executorService //used for the runAsync
//    def backgroundService
    def deployImagesService
    def grailsApplication
    def cytomineService
    def fileSystemService

    /**
     * Upload a file on Cytomine.
     * It will create all intermediate files and final abstract images.
     *
     * @param user The Cytomine user doing the request
     * @param storage The Cytomine virtual storage in which the file is uploaded
     * @param filename The filename of the uploaded file
     * @param filePath The path where the file is currently stored
     * @param isSync Whether the HTTP response waits for the end of deployment
     * @param projects A list of Cytomine projects to which the final abstract images are linked
     * @param properties A map of custom properties to associate to the final abstract images
     *
     * @return A map with the root UploadedFile and all the generated AbstractImages
     */
    def upload(User user, Storage storage, String filename, def filePath, boolean isSync, def projects, def properties) {
        if (!filePath) {
            throw new FileNotFoundException("Got an invalid file. Disk can be full.")
        }

        def temporaryFile = new File(filePath)
        if (!temporaryFile.exists()) {
            throw new FileNotFoundException(temporaryFile.absolutePath + " NOT EXIST!")
        }

        def imsServer = cytomineService.getThisImageServer()
        if (!imsServer)
            throw new MiddlewareException("IMS reference not found in core.")

        log.info "ImageServer: $imsServer"

        log.info "Create an uploadedFile instance and copy it to its storages"
        log.info "filePath=$filePath"
        log.info "absoluteFilePath=${temporaryFile.absolutePath}"

        // Root UF
        def size = temporaryFile.size()
        def extension = FilesUtils.getExtensionFromFilename(filename).toLowerCase()
        def status = UploadedFile.Status.UPLOADED
        def destinationPath = Paths.get(new Date().getTime().toString(), FilesUtils.correctFilename(filename))
        def uploadedFile = new UploadedFile(imsServer, filename, destinationPath.toString(), size, extension,
                UNDERTERMINED_CONTENT, projects, storage, user, status, null).save()

        File file = new File((String) uploadedFile.get('path'))
        fileSystemService.move(temporaryFile, file)

        def result = [:]
        result.uploadedFile = uploadedFile

        def uploadInfo = [
                user      : user,
                storage   : storage,
                imsServer : imsServer,
                isSync    : isSync,
                projects  : projects,
                properties: properties
        ]

        if (isSync) {
            log.info "Sync upload"
            deployFile(file, uploadedFile, uploadInfo, result)
        } else {
//            runAsync {
                log.info "Async upload"
                deployFile(file, uploadedFile, uploadInfo, result)
//            }
        }

        return result
    }

    /**
     * Deploy the received file. It results in a set of abstract images on which custom properties are added and image instances are created for the given projects.
     *
     * At the end, the uploaded file has the status
     * - DEPLOYED if everything happened correctly
     * - ERROR_DEPLOYMENT otherwise
     *
     * @param currentFile The physical representation of uploaded file
     * @param uploadedFile The Cytomine representation of uploaded file
     * @param uploadInfo A map with information about the upload
     * @param result A map filled with deployment results (method output)
     * @return
     */
    private def deployFile(File currentFile, UploadedFile uploadedFile, def uploadInfo, def result) {
        try {
            def file = new CytomineFile(currentFile.absolutePath)
            result.images = deploy(file, uploadedFile, null, null, uploadInfo)

            // Add properties to resulting abstract images
            result.images.each { deployed ->
                def props = []
                uploadInfo.properties.each {
                    props << new Property(deployed.image, (String) it.key, (String) it.value)
                }
                if (props.size() > 0) {
                    log.info "Add custom ${props.size()} properties to ${deployed.image}"
                    log.debug properties
                    props.each { it.save() } // TODO: replace by a multiple add in a single request
                }
            }

            // Add images to projects
            for (int i = 0; i < uploadInfo.projects.size(); i++) {
                def project = uploadInfo.projects.get(i)
                result.images.each { deployed ->
                    log.info "Adding ${deployed.image} to $project"
                    new ImageInstance(deployed.image, project).save()
                }
            }

            // Set uploaded file as deployed
            uploadedFile = new UploadedFile().fetch(uploadedFile.id)
            uploadedFile.changeStatus(UploadedFile.Status.DEPLOYED)

        } catch (DeploymentException | CytomineException e) {
            uploadedFile = new UploadedFile().fetch(uploadedFile.id)

            // Errors have an odd code.
            if (uploadedFile.get("status") % 2 != 0) {
                uploadedFile.changeStatus(UploadedFile.Status.ERROR_DEPLOYMENT)
            }

            if (uploadInfo.isSync) {
                throw new DeploymentException(e.getMessage())
            } else {
                e.printStackTrace()
            }
        }
    }


    /**
     * Recursively detect, convert and deploy the current file into Cytomine until a conversion leads to a format natively supported by Cytomine.
     *
     * @param currentFile The "physical" representation of the current file to deploy
     * @param uploadedFile The Cytomine representation of current file. If null, it is generated.
     * @param uploadedFileParent The Cytomine representation of the current file's parent
     * @param abstractImage The Cytomine representation of the image. Null until the file is an image.
     * @param uploadInfo A map with information about the upload
     *
     * @return A list of map [image: AbstractImage, slices: [AbstractSlice]]
     */
    private def deploy(CytomineFile currentFile, UploadedFile uploadedFile, UploadedFile uploadedFileParent, AbstractImage abstractImage, def uploadInfo) {

        log.info "Deploy $currentFile"
        def result = []

        def identifier = new FormatIdentifier(new CytomineFile(currentFile.absolutePath))

        // If the current file is a regular folder, recursively deploy its children but do not create an UF
        if (identifier.isClassicFolder()) {
            def errors = []
            currentFile.listFiles().each {
                try {
                    def child = new CytomineFile(it.absolutePath)
                    def deployed = deploy(child, null, uploadedFile ?: uploadedFileParent, abstractImage, uploadInfo)
                    result.addAll(deployed)
                } catch (DeploymentException e) {
                    errors << e.getMessage()
                }
            }

            if (!errors.isEmpty()) {
                throw new DeploymentException(errors.join("\n"))
            }

            return result
        }

        // Create a Cytomine representation for the current file if not yet done
        if (uploadedFile == null) {<commit_msg>Ignore Mac OS related files in zip<commit_after>package be.cytomine.server

/*
 * Copyright (c) 2009-2019. Authors: see NOTICE file.
 *
 * Licensed under the GNU Lesser General Public License, Version 2.1 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      https://www.gnu.org/licenses/lgpl-2.1.txt
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import be.cytomine.client.CytomineException
import be.cytomine.client.models.*
import be.cytomine.exception.DeploymentException
import be.cytomine.exception.FormatException
import be.cytomine.exception.MiddlewareException
import be.cytomine.formats.*
import be.cytomine.formats.archive.ArchiveFormat
import be.cytomine.formats.supported.NativeFormat
import utils.FilesUtils

import java.nio.file.Paths

class UploadService {

    final static public String UNDERTERMINED_CONTENT = "undetermined"

    def executorService //used for the runAsync
//    def backgroundService
    def deployImagesService
    def grailsApplication
    def cytomineService
    def fileSystemService

    /**
     * Upload a file on Cytomine.
     * It will create all intermediate files and final abstract images.
     *
     * @param user The Cytomine user doing the request
     * @param storage The Cytomine virtual storage in which the file is uploaded
     * @param filename The filename of the uploaded file
     * @param filePath The path where the file is currently stored
     * @param isSync Whether the HTTP response waits for the end of deployment
     * @param projects A list of Cytomine projects to which the final abstract images are linked
     * @param properties A map of custom properties to associate to the final abstract images
     *
     * @return A map with the root UploadedFile and all the generated AbstractImages
     */
    def upload(User user, Storage storage, String filename, def filePath, boolean isSync, def projects, def properties) {
        if (!filePath) {
            throw new FileNotFoundException("Got an invalid file. Disk can be full.")
        }

        def temporaryFile = new File(filePath)
        if (!temporaryFile.exists()) {
            throw new FileNotFoundException(temporaryFile.absolutePath + " NOT EXIST!")
        }

        def imsServer = cytomineService.getThisImageServer()
        if (!imsServer)
            throw new MiddlewareException("IMS reference not found in core.")

        log.info "ImageServer: $imsServer"

        log.info "Create an uploadedFile instance and copy it to its storages"
        log.info "filePath=$filePath"
        log.info "absoluteFilePath=${temporaryFile.absolutePath}"

        // Root UF
        def size = temporaryFile.size()
        def extension = FilesUtils.getExtensionFromFilename(filename).toLowerCase()
        def status = UploadedFile.Status.UPLOADED
        def destinationPath = Paths.get(new Date().getTime().toString(), FilesUtils.correctFilename(filename))
        def uploadedFile = new UploadedFile(imsServer, filename, destinationPath.toString(), size, extension,
                UNDERTERMINED_CONTENT, projects, storage, user, status, null).save()

        File file = new File((String) uploadedFile.get('path'))
        fileSystemService.move(temporaryFile, file)

        def result = [:]
        result.uploadedFile = uploadedFile

        def uploadInfo = [
                user      : user,
                storage   : storage,
                imsServer : imsServer,
                isSync    : isSync,
                projects  : projects,
                properties: properties
        ]

        if (isSync) {
            log.info "Sync upload"
            deployFile(file, uploadedFile, uploadInfo, result)
        } else {
//            runAsync {
                log.info "Async upload"
                deployFile(file, uploadedFile, uploadInfo, result)
//            }
        }

        return result
    }

    /**
     * Deploy the received file. It results in a set of abstract images on which custom properties are added and image instances are created for the given projects.
     *
     * At the end, the uploaded file has the status
     * - DEPLOYED if everything happened correctly
     * - ERROR_DEPLOYMENT otherwise
     *
     * @param currentFile The physical representation of uploaded file
     * @param uploadedFile The Cytomine representation of uploaded file
     * @param uploadInfo A map with information about the upload
     * @param result A map filled with deployment results (method output)
     * @return
     */
    private def deployFile(File currentFile, UploadedFile uploadedFile, def uploadInfo, def result) {
        try {
            def file = new CytomineFile(currentFile.absolutePath)
            result.images = deploy(file, uploadedFile, null, null, uploadInfo)

            // Add properties to resulting abstract images
            result.images.each { deployed ->
                def props = []
                uploadInfo.properties.each {
                    props << new Property(deployed.image, (String) it.key, (String) it.value)
                }
                if (props.size() > 0) {
                    log.info "Add custom ${props.size()} properties to ${deployed.image}"
                    log.debug properties
                    props.each { it.save() } // TODO: replace by a multiple add in a single request
                }
            }

            // Add images to projects
            for (int i = 0; i < uploadInfo.projects.size(); i++) {
                def project = uploadInfo.projects.get(i)
                result.images.each { deployed ->
                    log.info "Adding ${deployed.image} to $project"
                    new ImageInstance(deployed.image, project).save()
                }
            }

            // Set uploaded file as deployed
            uploadedFile = new UploadedFile().fetch(uploadedFile.id)
            uploadedFile.changeStatus(UploadedFile.Status.DEPLOYED)

        } catch (DeploymentException | CytomineException e) {
            uploadedFile = new UploadedFile().fetch(uploadedFile.id)

            // Errors have an odd code.
            if (uploadedFile.get("status") % 2 != 0) {
                uploadedFile.changeStatus(UploadedFile.Status.ERROR_DEPLOYMENT)
            }

            if (uploadInfo.isSync) {
                throw new DeploymentException(e.getMessage())
            } else {
                e.printStackTrace()
            }
        }
    }


    /**
     * Recursively detect, convert and deploy the current file into Cytomine until a conversion leads to a format natively supported by Cytomine.
     *
     * @param currentFile The "physical" representation of the current file to deploy
     * @param uploadedFile The Cytomine representation of current file. If null, it is generated.
     * @param uploadedFileParent The Cytomine representation of the current file's parent
     * @param abstractImage The Cytomine representation of the image. Null until the file is an image.
     * @param uploadInfo A map with information about the upload
     *
     * @return A list of map [image: AbstractImage, slices: [AbstractSlice]]
     */
    private def deploy(CytomineFile currentFile, UploadedFile uploadedFile, UploadedFile uploadedFileParent, AbstractImage abstractImage, def uploadInfo) {

        log.info "Deploy $currentFile"
        def result = []

        def identifier = new FormatIdentifier(new CytomineFile(currentFile.absolutePath))

        // If the current file is a regular folder, recursively deploy its children but do not create an UF
        if (identifier.isClassicFolder()) {
            def errors = []
            currentFile.listFiles().each {
                if (it.name == ".DS_STORE" || it.name.startsWith("__MACOSX"))
                    return

                try {
                    def child = new CytomineFile(it.absolutePath)
                    def deployed = deploy(child, null, uploadedFile ?: uploadedFileParent, abstractImage, uploadInfo)
                    result.addAll(deployed)
                } catch (DeploymentException e) {
                    errors << e.getMessage()
                }
            }

            if (!errors.isEmpty()) {
                throw new DeploymentException(errors.join("\n"))
            }

            return result
        }

        // Create a Cytomine representation for the current file if not yet done
        if (uploadedFile == null) {
{'fraction_non_alphanumeric': 0.07464312331594336, 'fraction_numerical': 0.001834546809608439, 'mean_word_length': 2.262390125303909, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1691040', 'n_tokens_mistral': 4496, 'n_tokens_neox': 4249, 'n_words': 1525}