<commit_before>package org.restheart.db;

/*
 * RESTHeart - the Web API for MongoDB
 * Copyright (C) SoftInstigate Srl
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
import com.mongodb.DuplicateKeyException;
import com.mongodb.MongoClient;
import com.mongodb.MongoGridFSException;
import com.mongodb.client.gridfs.GridFSBucket;
import com.mongodb.client.gridfs.GridFSBuckets;
import com.mongodb.client.gridfs.model.GridFSFile;
import com.mongodb.client.gridfs.model.GridFSUploadOptions;
import static com.mongodb.client.model.Filters.eq;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.Objects;
import org.bson.BsonDocument;
import org.bson.BsonObjectId;
import org.bson.BsonValue;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.restheart.utils.HttpStatus;

/**
 *
 * @author <NAME> {@literal <<EMAIL>>}
 */
public class GridFsDAO implements GridFsRepository {

    private static final String FILENAME = "filename";

    private static String extractBucketName(final String collectionName) {
        return collectionName.split("\\.")[0];
    }

    private final MongoClient client;

    public GridFsDAO() {
        client = MongoDBClientSingleton.getInstance().getClient();
    }

    @Override
    public OperationResult createFile(
            final Database db,
            final String dbName,
            final String bucketName,
            final BsonDocument metadata,
            final Path filePath)
            throws IOException, DuplicateKeyException {

        final String bucket = extractBucketName(bucketName);

        GridFSBucket gridFSBucket = GridFSBuckets.create(
                db.getDatabase(dbName),
                bucket);

        String filename = extractFilenameFromProperties(metadata);

        //add etag to metadata
        ObjectId etag = new ObjectId();
        metadata.put("_etag", new BsonObjectId(etag));

        try (InputStream sourceStream = new FileInputStream(filePath.toFile())) {

            if (metadata.get("_id") == null) {
                GridFSUploadOptions options = new GridFSUploadOptions()
                        .metadata(Document.parse(metadata.toJson()));

                ObjectId _id = gridFSBucket.uploadFromStream(
                        filename,
                        sourceStream,
                        options);

                return new OperationResult(HttpStatus.SC_CREATED,
                        new BsonObjectId(etag),
                        new BsonObjectId(_id));
            } else {
                BsonValue _id = metadata.remove("_id");

                GridFSUploadOptions options = new GridFSUploadOptions()
                        .metadata(Document.parse(metadata.toJson()));

                gridFSBucket.uploadFromStream(
                        _id,
                        filename,
                        sourceStream,
                        options);

                return new OperationResult(HttpStatus.SC_CREATED,
                        new BsonObjectId(etag),
                        _id);
            }
        }
    }

    private String extractFilenameFromProperties(
            final BsonDocument properties) {
        String filename = null;

        if (properties != null && properties.containsKey(FILENAME)) {
            BsonValue _filename = properties.get(FILENAME);

            if (_filename != null && _filename.isString()) {
                filename = _filename.asString().getValue();
            }
        }

        if (filename == null) {
            return "file";
        } else {
            return filename;
        }
    }

    @Override
    public OperationResult deleteFile(
            final Database db,
            final String dbName,
            final String bucketName,
            final BsonValue fileId,
            final String requestEtag,
            final boolean checkEtag) {
        final String bucket = extractBucketName(bucketName);

        GridFSBucket gridFSBucket = GridFSBuckets.create(
                db.getDatabase(dbName),
                bucket);

        GridFSFile file = gridFSBucket
                .find(eq("_id", fileId))
                .limit(1).iterator().tryNext();

        if (file == null) {
            return new OperationResult(HttpStatus.SC_NOT_FOUND);
        }
        if (checkEtag) {
            Document metadata = file.getMetadata();
            if (metadata != null) {
                Object oldEtag = metadata.get("_etag");

                if (oldEtag != null) {
                    if (requestEtag == null) {
                        return new OperationResult(HttpStatus.SC_CONFLICT, oldEtag);
                    } else if (!Objects.equals(oldEtag.toString(), requestEtag)) {
                        return new OperationResult(
                                HttpStatus.SC_PRECONDITION_FAILED, oldEtag);
                    }
                }
            }
        }

        // Even checking the existence of file before,
        // It's possible that the is already deleted at this point (due to concurrency problems).
        try {
            gridFSBucket.delete(fileId);
        } catch (MongoGridFSException e) {
            return new OperationResult(HttpStatus.SC_NOT_FOUND);
        }

        return new OperationResult(HttpStatus.SC_NO_CONTENT);
    }

    @Override
    public void deleteChunksCollection(final Database db,
            final String dbName,
            final String bucketName
    ) {
        String chunksCollName = extractBucketName(bucketName).concat(".chunks");
        client.getDatabase(dbName).getCollection(chunksCollName).drop();
    }

}<commit_msg>Try to avoid concurrent deletions of the same file<commit_after>package org.restheart.db;

/*
 * RESTHeart - the Web API for MongoDB
 * Copyright (C) SoftInstigate Srl
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
import com.mongodb.DuplicateKeyException;
import com.mongodb.MongoClient;
import com.mongodb.MongoGridFSException;
import com.mongodb.client.gridfs.GridFSBucket;
import com.mongodb.client.gridfs.GridFSBuckets;
import com.mongodb.client.gridfs.model.GridFSFile;
import com.mongodb.client.gridfs.model.GridFSUploadOptions;
import static com.mongodb.client.model.Filters.eq;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.Objects;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.Logger;
import org.bson.BsonDocument;
import org.bson.BsonObjectId;
import org.bson.BsonValue;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.restheart.utils.HttpStatus;
import org.slf4j.LoggerFactory;

/**
 *
 * @author <NAME> {@literal <<EMAIL>>}
 */
public class GridFsDAO implements GridFsRepository {

    private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(GridFsDAO.class);

    private static final String FILENAME = "filename";

    private static String extractBucketName(final String collectionName) {
        return collectionName.split("\\.")[0];
    }

    private final MongoClient client;

    private final Lock deleteLock = new ReentrantLock();

    public GridFsDAO() {
        client = MongoDBClientSingleton.getInstance().getClient();
    }

    /**
     *
     * @param db
     * @param dbName
     * @param bucketName
     * @param metadata
     * @param filePath
     * @return the OperationResult
     * @throws IOException
     * @throws DuplicateKeyException
     */
    @Override
    public OperationResult createFile(
            final Database db,
            final String dbName,
            final String bucketName,
            final BsonDocument metadata,
            final Path filePath)
            throws IOException, DuplicateKeyException {

        final String bucket = extractBucketName(bucketName);

        GridFSBucket gridFSBucket = GridFSBuckets.create(
                db.getDatabase(dbName),
                bucket);

        String filename = extractFilenameFromProperties(metadata);

        //add etag to metadata
        ObjectId etag = new ObjectId();
        metadata.put("_etag", new BsonObjectId(etag));

        try (InputStream sourceStream = new FileInputStream(filePath.toFile())) {

            if (metadata.get("_id") == null) {
                GridFSUploadOptions options = new GridFSUploadOptions()
                        .metadata(Document.parse(metadata.toJson()));

                ObjectId _id = gridFSBucket.uploadFromStream(
                        filename,
                        sourceStream,
                        options);

                return new OperationResult(HttpStatus.SC_CREATED,
                        new BsonObjectId(etag),
                        new BsonObjectId(_id));
            } else {
                BsonValue _id = metadata.remove("_id");

                GridFSUploadOptions options = new GridFSUploadOptions()
                        .metadata(Document.parse(metadata.toJson()));

                gridFSBucket.uploadFromStream(
                        _id,
                        filename,
                        sourceStream,
                        options);

                return new OperationResult(HttpStatus.SC_CREATED,
                        new BsonObjectId(etag),
                        _id);
            }
        }
    }

    private String extractFilenameFromProperties(
            final BsonDocument properties) {
        String filename = null;

        if (properties != null && properties.containsKey(FILENAME)) {
            BsonValue _filename = properties.get(FILENAME);

            if (_filename != null && _filename.isString()) {
                filename = _filename.asString().getValue();
            }
        }

        if (filename == null) {
            return "file";
        } else {
            return filename;
        }
    }

    /**
     *
     * @param db
     * @param dbName
     * @param bucketName
     * @param fileId
     * @param requestEtag
     * @param checkEtag
     * @return the OperationResult
     */
    @Override
    public OperationResult deleteFile(
            final Database db,
            final String dbName,
            final String bucketName,
            final BsonValue fileId,
            final String requestEtag,
            final boolean checkEtag) {

        final String bucket = extractBucketName(bucketName);

        GridFSBucket gridFSBucket = GridFSBuckets.create(
                db.getDatabase(dbName),
                bucket);

        // try to avoid concurrent deletions of the same file as much as possible
        // Note: this won't help much if RESTHeart is clustered, as the lock is local
        deleteLock.lock();
        try {
            GridFSFile file = gridFSBucket
                    .find(eq("_id", fileId))
                    .limit(1).iterator().tryNext();

            if (file == null) {
                return new OperationResult(HttpStatus.SC_NOT_FOUND);
            }

            if (checkEtag) {
                Document metadata = file.getMetadata();
                if (metadata != null) {
                    Object oldEtag = metadata.get("_etag");

                    if (oldEtag != null) {
                        if (requestEtag == null) {
                            return new OperationResult(HttpStatus.SC_CONFLICT, oldEtag);
                        } else if (!Objects.equals(oldEtag.toString(), requestEtag)) {
                            return new OperationResult(
                                    HttpStatus.SC_PRECONDITION_FAILED, oldEtag);
                        }
                    }
                }
            }

            try {
                gridFSBucket.delete(fileId);
                LOGGER.info("Succesfully deleted fileId {}", fileId);
            } catch (MongoGridFSException e) {
                LOGGER.error("Can't delete fileId '{}'", fileId, e);
                return new OperationResult(HttpStatus.SC_NOT_FOUND);
            }

            return new OperationResult(HttpStatus.SC_NO_CONTENT);

        } finally {
            deleteLock.unlock();
        }
    }

    /**
     *
     * @param db
     * @param dbName
     * @param bucketName
     */
    @Override
    public void deleteChunksCollection(final Database db,
            final String dbName,
            final String bucketName
    ) {
        String chunksCollName = extractBucketName(bucketName).concat(".chunks");
        client.getDatabase(dbName).getCollection(chunksCollName).drop();
    }

}
{'fraction_non_alphanumeric': 0.08339952343129468, 'fraction_numerical': 0.0005776590367535563, 'mean_word_length': 1.9695540308747856, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '244594', 'n_tokens_mistral': 3625, 'n_tokens_neox': 3429, 'n_words': 937}