<commit_before>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage.testing;

import com.google.common.collect.ImmutableMap;
import com.google.gcloud.AuthCredentials;
import com.google.gcloud.storage.BlobInfo;
import com.google.gcloud.RetryParams;
import com.google.gcloud.storage.Storage;
import com.google.gcloud.storage.StorageException;
import com.google.gcloud.storage.StorageOptions;
import com.google.gcloud.storage.testing.RemoteGcsHelper.Option.KeyFromClasspath;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * Utility to create a remote storage configuration for testing
 */
public class RemoteGcsHelper {

  private static final Logger log = Logger.getLogger(RemoteGcsHelper.class.getName());
  private static final String BUCKET_NAME_PREFIX = "gcloud-test-bucket-temp-";
  private static final String PROJECT_ID_ENV_VAR = "GCLOUD_TESTS_PROJECT_ID";
  private static final String PRIVATE_KEY_ENV_VAR = "GCLOUD_TESTS_KEY";

  private final StorageOptions options;

  private RemoteGcsHelper(StorageOptions options) {
    this.options = options;
  }

  /**
   * Returns a {@link StorageOptions} object to be used for testing.
   */
  public StorageOptions options() {
    return options;
  }

  /**
   * Deletes a bucket, even if non-empty. Objects in the bucket are listed and deleted until bucket
   * deletion succeeds or {@code timeout} expires.
   *
   * @param storage the storage service to be used to issue requests
   * @param bucket the bucket to be deleted
   * @param timeout the maximum time to wait
   * @param unit the time unit of the timeout argument
   * @return true if deletion succeeded, false if timeout expired.
   * @throws InterruptedException if the thread deleting the bucket is interrupted while waiting
   * @throws ExecutionException if an exception was thrown while deleting bucket or bucket objects
   */
  public static Boolean forceDelete(Storage storage, String bucket, long timeout, TimeUnit unit)
      throws InterruptedException, ExecutionException {
    ExecutorService executor = Executors.newSingleThreadExecutor();
    Future<Boolean> future = executor.submit(new DeleteBucketTask(storage, bucket));
    try {
      return future.get(timeout, unit);
    } catch (TimeoutException ex) {
      return false;
    } finally {
      executor.shutdown();
    }
  }

  /**
   * Returns a bucket name generated using a random UUID.
   */
  public static String generateBucketName() {
    return BUCKET_NAME_PREFIX + UUID.randomUUID().toString();
  }

  /**
   * Creates a {@code RemoteGcsHelper} object for the given project id and JSON key path.
   *
   * @param projectId id of the project to be used for running the tests
   * @param keyPath path to the JSON key to be used for running the tests
   * @param options creation options
   * @return A {@code RemoteGcsHelper} object for the provided options.
   * @throws com.google.gcloud.storage.testing.RemoteGcsHelper.GcsHelperException if the file pointed by 
   * {@code keyPath} does not exist
   */
  public static RemoteGcsHelper create(String projectId, String keyPath, Option... options)
      throws GcsHelperException {
    boolean keyFromClassPath = false;
    Map<Class<? extends Option>, Option> optionsMap = Option.asImmutableMap(options);
    if (optionsMap.containsKey(KeyFromClasspath.class)) {
      keyFromClassPath =
          ((KeyFromClasspath) optionsMap.get(KeyFromClasspath.class)).keyFromClasspath();
    }
    try {
      InputStream keyFileStream;
      if (keyFromClassPath) {
        keyFileStream = RemoteGcsHelper.class.getResourceAsStream(keyPath);
        if (keyFileStream == null) {
          throw new FileNotFoundException(keyPath + " not found in classpath");
        }
      } else {
        keyFileStream = new FileInputStream(keyPath);
      }
      StorageOptions storageOptions = StorageOptions.builder()
          .authCredentials(AuthCredentials.createForJson(keyFileStream))
          .projectId(projectId)
          .retryParams(RetryParams.getDefaultInstance())
          .build();
      return new RemoteGcsHelper(storageOptions);
    } catch (FileNotFoundException ex) {
      if (log.isLoggable(Level.WARNING)) {
        log.log(Level.WARNING, ex.getMessage());
      }
      throw GcsHelperException.translate(ex);
    } catch (IOException ex) {
      if (log.isLoggable(Level.WARNING)) {
        log.log(Level.WARNING, ex.getMessage());
      }<commit_msg>Increase retry parameters for storage<commit_after>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage.testing;

import com.google.common.collect.ImmutableMap;
import com.google.gcloud.AuthCredentials;
import com.google.gcloud.storage.BlobInfo;
import com.google.gcloud.RetryParams;
import com.google.gcloud.storage.Storage;
import com.google.gcloud.storage.StorageException;
import com.google.gcloud.storage.StorageOptions;
import com.google.gcloud.storage.testing.RemoteGcsHelper.Option.KeyFromClasspath;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * Utility to create a remote storage configuration for testing
 */
public class RemoteGcsHelper {

  private static final Logger log = Logger.getLogger(RemoteGcsHelper.class.getName());
  private static final String BUCKET_NAME_PREFIX = "gcloud-test-bucket-temp-";
  private static final String PROJECT_ID_ENV_VAR = "GCLOUD_TESTS_PROJECT_ID";
  private static final String PRIVATE_KEY_ENV_VAR = "GCLOUD_TESTS_KEY";
  private final StorageOptions options;

  private RemoteGcsHelper(StorageOptions options) {
    this.options = options;
  }

  /**
   * Returns a {@link StorageOptions} object to be used for testing.
   */
  public StorageOptions options() {
    return options;
  }

  /**
   * Deletes a bucket, even if non-empty. Objects in the bucket are listed and deleted until bucket
   * deletion succeeds or {@code timeout} expires.
   *
   * @param storage the storage service to be used to issue requests
   * @param bucket the bucket to be deleted
   * @param timeout the maximum time to wait
   * @param unit the time unit of the timeout argument
   * @return true if deletion succeeded, false if timeout expired.
   * @throws InterruptedException if the thread deleting the bucket is interrupted while waiting
   * @throws ExecutionException if an exception was thrown while deleting bucket or bucket objects
   */
  public static Boolean forceDelete(Storage storage, String bucket, long timeout, TimeUnit unit)
      throws InterruptedException, ExecutionException {
    ExecutorService executor = Executors.newSingleThreadExecutor();
    Future<Boolean> future = executor.submit(new DeleteBucketTask(storage, bucket));
    try {
      return future.get(timeout, unit);
    } catch (TimeoutException ex) {
      return false;
    } finally {
      executor.shutdown();
    }
  }

  /**
   * Returns a bucket name generated using a random UUID.
   */
  public static String generateBucketName() {
    return BUCKET_NAME_PREFIX + UUID.randomUUID().toString();
  }

  /**
   * Creates a {@code RemoteGcsHelper} object for the given project id and JSON key path.
   *
   * @param projectId id of the project to be used for running the tests
   * @param keyPath path to the JSON key to be used for running the tests
   * @param options creation options
   * @return A {@code RemoteGcsHelper} object for the provided options.
   * @throws com.google.gcloud.storage.testing.RemoteGcsHelper.GcsHelperException if the file pointed by 
   * {@code keyPath} does not exist
   */
  public static RemoteGcsHelper create(String projectId, String keyPath, Option... options)
      throws GcsHelperException {
    boolean keyFromClassPath = false;
    Map<Class<? extends Option>, Option> optionsMap = Option.asImmutableMap(options);
    if (optionsMap.containsKey(KeyFromClasspath.class)) {
      keyFromClassPath =
          ((KeyFromClasspath) optionsMap.get(KeyFromClasspath.class)).keyFromClasspath();
    }
    try {
      InputStream keyFileStream;
      if (keyFromClassPath) {
        keyFileStream = RemoteGcsHelper.class.getResourceAsStream(keyPath);
        if (keyFileStream == null) {
          throw new FileNotFoundException(keyPath + " not found in classpath");
        }
      } else {
        keyFileStream = new FileInputStream(keyPath);
      }
      StorageOptions storageOptions = StorageOptions.builder()
          .authCredentials(AuthCredentials.createForJson(keyFileStream))
          .projectId(projectId)
          .retryParams(RetryParams.builder()
              .retryMaxAttempts(10)
              .retryMinAttempts(6)
              .maxRetryDelayMillis(30000)
              .totalRetryPeriodMillis(120000)
              .initialRetryDelayMillis(250)
              .build())
          .build();
      return new RemoteGcsHelper(storageOptions);
    } catch (FileNotFoundException ex) {
      if (log.isLoggable(Level.WARNING)) {
        log.log(Level.WARNING, ex.getMessage());
      }
      throw GcsHelperException.translate(ex);
    } catch (IOException ex) {
      if (log.isLoggable(Level.WARNING)) {
        log.log(Level.WARNING, ex.getMessage());
      }
{'fraction_non_alphanumeric': 0.08553100498930863, 'fraction_numerical': 0.002940128296507484, 'mean_word_length': 3.716386554621849, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4268455', 'n_tokens_mistral': 3040, 'n_tokens_neox': 3011, 'n_words': 952}