<commit_before>/*
 * Copyright 2017, TeamDev Ltd. All rights reserved.
 *
 * Redistribution and use in source and/or binary forms, with or without
 * modification, must retain the above copyright notice and the following
 * disclaimer.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.spine3.tools.gcs;

import com.google.api.gax.paging.Page;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.Bucket;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobListOption;
import com.google.cloud.storage.StorageOptions;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Ordering;
import org.gradle.api.DefaultTask;
import org.gradle.api.tasks.TaskAction;
import org.joda.time.DateTime;
import org.json.JSONObject;
import org.slf4j.Logger;

import javax.annotation.Nullable;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.lang.String.format;

/**
 * The task, that deletes the specified folder if it's age exceeds {@link CleaningThreshold}.
 *
 * <p>Configuration example:
 * <pre>{@code
 * cleanGCS {
 *      keyFile = "gcs-key-file.json"
 *      bucketName = "example.com"
 *      targetFolder = "trash"
 *      threshold = 30
 * }
 * }</pre>
 *
 * <p>All task parameters are mandatory.
 *
 * @author <NAME>
 */
public class CleanGcsTask extends DefaultTask {

    private static final String PROJECT_ID_KEY = "project_id";

    /**
     * A relative path to a file that contains service account credentials in JSON format.
     *
     * <p>Path should starts from a project root.
     */
    private String keyFile;

    /**
     * A name of bucket containing {@link #targetFolder}.
     */
    private String bucketName;

    /**
     * A name of folder to delete if {@link #threshold} exceeded.
     */
    private String targetFolder;

    /**
     * Cleaning threshold measurable in days.
     */
    private CleaningThreshold threshold;

    @TaskAction
    void cleanGcs() {
        checkParameters();
        final Storage storage = getStorage();
        checkBucketExists(storage);

        final Page<Blob> blobs = storage.list(bucketName, BlobListOption.prefix(targetFolder));
        final Iterable<Blob> allBlobs = blobs.iterateAll();
        if (!allBlobs.iterator()
                     .hasNext()) {
            log().info("Folder `{}` is not exists. Nothing to clean.", targetFolder);
            return;
        }

        final DateTime oldestBlobCreation = getOldestBlobCreationDate(allBlobs);
        final DateTime cleaningTrigger = oldestBlobCreation.plus(threshold.toMillis());
        final boolean isCleaningRequired = cleaningTrigger.isBeforeNow();
        if (isCleaningRequired) {
            for (Blob blob : allBlobs) {
                storage.delete(blob.getBlobId());
            }
            log().info("Folder `{}` in bucketName `{}` deleted.", targetFolder, bucketName);
        } else {<commit_msg>Expand task Javadoc and fix grammar<commit_after>/*
 * Copyright 2017, TeamDev Ltd. All rights reserved.
 *
 * Redistribution and use in source and/or binary forms, with or without
 * modification, must retain the above copyright notice and the following
 * disclaimer.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.spine3.tools.gcs;

import com.google.api.gax.paging.Page;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.Bucket;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobListOption;
import com.google.cloud.storage.StorageOptions;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Ordering;
import org.gradle.api.DefaultTask;
import org.gradle.api.tasks.TaskAction;
import org.joda.time.DateTime;
import org.json.JSONObject;
import org.slf4j.Logger;

import javax.annotation.Nullable;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.lang.String.format;

/**
 * The task, that deletes the specified folder from Google Cloud Storage.
 *
 * The folder will be deleted if its age is over than {@link CleaningThreshold}.
 *
 * <p>Configuration example:
 * <pre>{@code
 * cleanGCS {
 *      keyFile = "gcs-key-file.json"
 *      bucketName = "example.com"
 *      targetFolder = "trash"
 *      threshold = 30
 * }
 * }</pre>
 *
 * <p>All task parameters are mandatory.
 *
 * @author <NAME>
 */
public class CleanGcsTask extends DefaultTask {

    private static final String PROJECT_ID_KEY = "project_id";

    /**
     * A relative path to a file that contains service account credentials in JSON format.
     *
     * <p>Path should starts from a project root.
     */
    private String keyFile;

    /**
     * A name of bucket containing {@link #targetFolder}.
     */
    private String bucketName;

    /**
     * A name of folder to delete if {@link #threshold} exceeded.
     */
    private String targetFolder;

    /**
     * A cleaning threshold measurable in days.
     *
     * <p>If a threshold exceeded, {@link #targetFolder} will be deleted.
     */
    private CleaningThreshold threshold;

    @TaskAction
    void cleanGcs() {
        checkParameters();
        final Storage storage = getStorage();
        checkBucketExists(storage);

        final Page<Blob> blobs = storage.list(bucketName, BlobListOption.prefix(targetFolder));
        final Iterable<Blob> allBlobs = blobs.iterateAll();
        if (!allBlobs.iterator()
                     .hasNext()) {
            log().info("Folder `{}` is not exists. Nothing to clean.", targetFolder);
            return;
        }

        final DateTime oldestBlobCreation = getOldestBlobCreationDate(allBlobs);
        final DateTime cleaningTrigger = oldestBlobCreation.plus(threshold.toMillis());
        final boolean isCleaningRequired = cleaningTrigger.isBeforeNow();
        if (isCleaningRequired) {
            for (Blob blob : allBlobs) {
                storage.delete(blob.getBlobId());
            }
            log().info("Folder `{}` in bucketName `{}` deleted.", targetFolder, bucketName);
        } else {
{'fraction_non_alphanumeric': 0.0962774342185037, 'fraction_numerical': 0.0021826118588577663, 'mean_word_length': 3.394246137453383, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '978760', 'n_tokens_mistral': 2343, 'n_tokens_neox': 2206, 'n_words': 738}