<commit_before>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.Page;
import com.google.gcloud.RestorableState;
import com.google.gcloud.storage.Storage.BlobField;
import com.google.gcloud.storage.Storage.BucketField;
import com.google.gcloud.storage.testing.RemoteGcsHelper;

import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;

public class ITStorageTest {

  private static Storage storage;

  private static final Logger log = Logger.getLogger(ITStorageTest.class.getName());
  private static final String BUCKET = RemoteGcsHelper.generateBucketName();
  private static final String CONTENT_TYPE = "text/plain";
  private static final byte[] BLOB_BYTE_CONTENT = {0xD, 0xE, 0xA, 0xD};
  private static final String BLOB_STRING_CONTENT = "Hello Google Cloud Storage!";

  @BeforeClass
  public static void beforeClass() {
    RemoteGcsHelper gcsHelper = RemoteGcsHelper.create();
    storage = gcsHelper.options().service();
    storage.create(BucketInfo.of(BUCKET));
  }

  @AfterClass
  public static void afterClass()
      throws ExecutionException, TimeoutException, InterruptedException {
    if (storage != null && !RemoteGcsHelper.forceDelete(storage, BUCKET, 5, TimeUnit.SECONDS)) {
      if (log.isLoggable(Level.WARNING)) {
        log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET);
      }
    }
  }

  @Test(timeout = 5000)
  public void testListBuckets() throws InterruptedException {
    Iterator<BucketInfo> bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
        Storage.BucketListOption.fields()).values().iterator();
    while (!bucketIterator.hasNext()) {
      Thread.sleep(500);
      bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
          Storage.BucketListOption.fields()).values().iterator();
    }
    while (bucketIterator.hasNext()) {
      BucketInfo remoteBucket = bucketIterator.next();
      assertTrue(remoteBucket.name().startsWith(BUCKET));
      assertNull(remoteBucket.createTime());
      assertNull(remoteBucket.selfLink());
    }
  }

  @Test
  public void testGetBucketSelectedFields() {
    BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields(BucketField.ID));
    assertEquals(BUCKET, remoteBucket.name());
    assertNull(remoteBucket.createTime());
    assertNotNull(remoteBucket.id());
  }

  @Test
  public void testGetBucketAllSelectedFields() {
    BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields(BucketField.ACL,
        BucketField.CORS, BucketField.DEFAULT_OBJECT_ACL, BucketField.ETAG, BucketField.ID,
        BucketField.LOCATION, BucketField.METAGENERATION, BucketField.NAME, BucketField.OWNER,
        BucketField.SELF_LINK, BucketField.STORAGE_CLASS, BucketField.TIME_CREATED,
        BucketField.VERSIONING, BucketField.WEBSITE));
    assertEquals(BUCKET, remoteBucket.name());
    assertNotNull(remoteBucket.createTime());
    assertNotNull(remoteBucket.selfLink());
  }

  @Test
  public void testGetBucketEmptyFields() {
    BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields());
    assertEquals(BUCKET, remoteBucket.name());
    assertNull(remoteBucket.createTime());
    assertNull(remoteBucket.selfLink());
  }

  @Test
  public void testCreateBlob() {
    String blobName = "test-create-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
    BlobInfo remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT);
    assertNotNull(remoteBlob);
    assertEquals(blob.blobId(), remoteBlob.blobId());
    byte[] readBytes = storage.readAllBytes(BUCKET, blobName);
    assertArrayEquals(BLOB_BYTE_CONTENT, readBytes);
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testCreateEmptyBlob() {
    String blobName = "test-create-empty-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
    BlobInfo remoteBlob = storage.create(blob);
    assertNotNull(remoteBlob);
    assertEquals(blob.blobId(), remoteBlob.blobId());
    byte[] readBytes = storage.readAllBytes(BUCKET, blobName);
    assertArrayEquals(new byte[0], readBytes);
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testCreateBlobStream() throws UnsupportedEncodingException {
    String blobName = "test-create-blob-stream";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).build();
    ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8));
    BlobInfo remoteBlob = storage.create(blob, stream);
    assertNotNull(remoteBlob);
    assertEquals(blob.blobId(), remoteBlob.blobId());
    assertEquals(blob.contentType(), remoteBlob.contentType());
    byte[] readBytes = storage.readAllBytes(BUCKET, blobName);
    assertEquals(BLOB_STRING_CONTENT, new String(readBytes, UTF_8));
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testCreateBlobFail() {
    String blobName = "test-create-blob-fail";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
    assertNotNull(storage.create(blob));
    try {
      storage.create(blob.toBuilder().generation(-1L).build(), BLOB_BYTE_CONTENT,
          Storage.BlobTargetOption.generationMatch());
      fail("StorageException was expected");
    } catch (StorageException ex) {
      // expected
    }
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testCreateBlobMd5Fail() throws UnsupportedEncodingException {
    String blobName = "test-create-blob-md5-fail";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName)
        .contentType(CONTENT_TYPE)
        .md5("O1R4G1HJSDUISJjoIYmVhQ==")
        .build();
    ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8));
    try {
      storage.create(blob, stream, Storage.BlobWriteOption.md5Match());
      fail("StorageException was expected");
    } catch (StorageException ex) {
      // expected
    }
  }

  @Test
  public void testGetBlobEmptySelectedFields() {
    String blobName = "test-get-empty-selected-fields-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).build();
    assertNotNull(storage.create(blob));
    BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields());
    assertEquals(blob.blobId(), remoteBlob.blobId());
    assertNull(remoteBlob.contentType());
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testGetBlobSelectedFields() {
    String blobName = "test-get-selected-fields-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName)
        .contentType(CONTENT_TYPE)
        .metadata(ImmutableMap.of("k", "v"))
        .build();
    assertNotNull(storage.create(blob));
    BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields(
        BlobField.METADATA));
    assertEquals(blob.blobId(), remoteBlob.blobId());
    assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata());
    assertNull(remoteBlob.contentType());
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testGetBlobAllSelectedFields() {
    String blobName = "test-get-all-selected-fields-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName)
        .contentType(CONTENT_TYPE)
        .metadata(ImmutableMap.of("k", "v"))
        .build();
    assertNotNull(storage.create(blob));
    BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields(
        BlobField.ACL, BlobField.BUCKET, BlobField.CACHE_CONTROL, BlobField.COMPONENT_COUNT,
        BlobField.CONTENT_DISPOSITION, BlobField.CONTENT_ENCODING, BlobField.CONTENT_LANGUAGE,
        BlobField.CONTENT_TYPE, BlobField.CRC32C, BlobField.ETAG, BlobField.GENERATION,
        BlobField.ID, BlobField.KIND, BlobField.MD5HASH, BlobField.MEDIA_LINK, BlobField.METADATA,
        BlobField.METAGENERATION, BlobField.NAME, BlobField.OWNER, BlobField.SELF_LINK,
        BlobField.SIZE, BlobField.STORAGE_CLASS, BlobField.TIME_DELETED, BlobField.UPDATED));
    assertEquals(blob.blobId(), remoteBlob.blobId());
    assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata());
    assertNotNull(remoteBlob.id());
    assertNotNull(remoteBlob.selfLink());
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testListBlobsSelectedFields() {
    String[] blobNames = {"test-list-blobs-selected-fields-blob1",
        "test-list-blobs-selected-fields-blob2"};
    ImmutableMap metadata = ImmutableMap.of("k", "v");
    BlobInfo blob1 = BlobInfo.builder(BUCKET, blobNames[0])
        .contentType(CONTENT_TYPE)
        .metadata(metadata)
        .build();
    BlobInfo blob2 = BlobInfo.builder(BUCKET, blobNames[1])
        .contentType(CONTENT_TYPE)
        .metadata(metadata)
        .build();
    assertNotNull(storage.create(blob1));
    assertNotNull(storage.create(blob2));<commit_msg>Use BlobField and BucketField .values() in IT tests<commit_after>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.Page;
import com.google.gcloud.RestorableState;
import com.google.gcloud.storage.Storage.BlobField;
import com.google.gcloud.storage.Storage.BucketField;
import com.google.gcloud.storage.testing.RemoteGcsHelper;

import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;

public class ITStorageTest {

  private static Storage storage;

  private static final Logger log = Logger.getLogger(ITStorageTest.class.getName());
  private static final String BUCKET = RemoteGcsHelper.generateBucketName();
  private static final String CONTENT_TYPE = "text/plain";
  private static final byte[] BLOB_BYTE_CONTENT = {0xD, 0xE, 0xA, 0xD};
  private static final String BLOB_STRING_CONTENT = "Hello Google Cloud Storage!";

  @BeforeClass
  public static void beforeClass() {
    RemoteGcsHelper gcsHelper = RemoteGcsHelper.create();
    storage = gcsHelper.options().service();
    storage.create(BucketInfo.of(BUCKET));
  }

  @AfterClass
  public static void afterClass()
      throws ExecutionException, TimeoutException, InterruptedException {
    if (storage != null && !RemoteGcsHelper.forceDelete(storage, BUCKET, 5, TimeUnit.SECONDS)) {
      if (log.isLoggable(Level.WARNING)) {
        log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET);
      }
    }
  }

  @Test(timeout = 5000)
  public void testListBuckets() throws InterruptedException {
    Iterator<BucketInfo> bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
        Storage.BucketListOption.fields()).values().iterator();
    while (!bucketIterator.hasNext()) {
      Thread.sleep(500);
      bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
          Storage.BucketListOption.fields()).values().iterator();
    }
    while (bucketIterator.hasNext()) {
      BucketInfo remoteBucket = bucketIterator.next();
      assertTrue(remoteBucket.name().startsWith(BUCKET));
      assertNull(remoteBucket.createTime());
      assertNull(remoteBucket.selfLink());
    }
  }

  @Test
  public void testGetBucketSelectedFields() {
    BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields(BucketField.ID));
    assertEquals(BUCKET, remoteBucket.name());
    assertNull(remoteBucket.createTime());
    assertNotNull(remoteBucket.id());
  }

  @Test
  public void testGetBucketAllSelectedFields() {
    BucketInfo remoteBucket = storage.get(BUCKET,
        Storage.BucketGetOption.fields(BucketField.values()));
    assertEquals(BUCKET, remoteBucket.name());
    assertNotNull(remoteBucket.createTime());
    assertNotNull(remoteBucket.selfLink());
  }

  @Test
  public void testGetBucketEmptyFields() {
    BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields());
    assertEquals(BUCKET, remoteBucket.name());
    assertNull(remoteBucket.createTime());
    assertNull(remoteBucket.selfLink());
  }

  @Test
  public void testCreateBlob() {
    String blobName = "test-create-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
    BlobInfo remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT);
    assertNotNull(remoteBlob);
    assertEquals(blob.blobId(), remoteBlob.blobId());
    byte[] readBytes = storage.readAllBytes(BUCKET, blobName);
    assertArrayEquals(BLOB_BYTE_CONTENT, readBytes);
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testCreateEmptyBlob() {
    String blobName = "test-create-empty-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
    BlobInfo remoteBlob = storage.create(blob);
    assertNotNull(remoteBlob);
    assertEquals(blob.blobId(), remoteBlob.blobId());
    byte[] readBytes = storage.readAllBytes(BUCKET, blobName);
    assertArrayEquals(new byte[0], readBytes);
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testCreateBlobStream() throws UnsupportedEncodingException {
    String blobName = "test-create-blob-stream";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).build();
    ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8));
    BlobInfo remoteBlob = storage.create(blob, stream);
    assertNotNull(remoteBlob);
    assertEquals(blob.blobId(), remoteBlob.blobId());
    assertEquals(blob.contentType(), remoteBlob.contentType());
    byte[] readBytes = storage.readAllBytes(BUCKET, blobName);
    assertEquals(BLOB_STRING_CONTENT, new String(readBytes, UTF_8));
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testCreateBlobFail() {
    String blobName = "test-create-blob-fail";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
    assertNotNull(storage.create(blob));
    try {
      storage.create(blob.toBuilder().generation(-1L).build(), BLOB_BYTE_CONTENT,
          Storage.BlobTargetOption.generationMatch());
      fail("StorageException was expected");
    } catch (StorageException ex) {
      // expected
    }
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testCreateBlobMd5Fail() throws UnsupportedEncodingException {
    String blobName = "test-create-blob-md5-fail";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName)
        .contentType(CONTENT_TYPE)
        .md5("O1R4G1HJSDUISJjoIYmVhQ==")
        .build();
    ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8));
    try {
      storage.create(blob, stream, Storage.BlobWriteOption.md5Match());
      fail("StorageException was expected");
    } catch (StorageException ex) {
      // expected
    }
  }

  @Test
  public void testGetBlobEmptySelectedFields() {
    String blobName = "test-get-empty-selected-fields-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).build();
    assertNotNull(storage.create(blob));
    BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields());
    assertEquals(blob.blobId(), remoteBlob.blobId());
    assertNull(remoteBlob.contentType());
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testGetBlobSelectedFields() {
    String blobName = "test-get-selected-fields-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName)
        .contentType(CONTENT_TYPE)
        .metadata(ImmutableMap.of("k", "v"))
        .build();
    assertNotNull(storage.create(blob));
    BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields(
        BlobField.METADATA));
    assertEquals(blob.blobId(), remoteBlob.blobId());
    assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata());
    assertNull(remoteBlob.contentType());
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testGetBlobAllSelectedFields() {
    String blobName = "test-get-all-selected-fields-blob";
    BlobInfo blob = BlobInfo.builder(BUCKET, blobName)
        .contentType(CONTENT_TYPE)
        .metadata(ImmutableMap.of("k", "v"))
        .build();
    assertNotNull(storage.create(blob));
    BlobInfo remoteBlob = storage.get(blob.blobId(),
        Storage.BlobGetOption.fields(BlobField.values()));
    assertEquals(blob.blobId(), remoteBlob.blobId());
    assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata());
    assertNotNull(remoteBlob.id());
    assertNotNull(remoteBlob.selfLink());
    assertTrue(storage.delete(BUCKET, blobName));
  }

  @Test
  public void testListBlobsSelectedFields() {
    String[] blobNames = {"test-list-blobs-selected-fields-blob1",
        "test-list-blobs-selected-fields-blob2"};
    ImmutableMap metadata = ImmutableMap.of("k", "v");
    BlobInfo blob1 = BlobInfo.builder(BUCKET, blobNames[0])
        .contentType(CONTENT_TYPE)
        .metadata(metadata)
        .build();
    BlobInfo blob2 = BlobInfo.builder(BUCKET, blobNames[1])
        .contentType(CONTENT_TYPE)
        .metadata(metadata)
        .build();
    assertNotNull(storage.create(blob1));
    assertNotNull(storage.create(blob2));
{'fraction_non_alphanumeric': 0.11828489379610833, 'fraction_numerical': 0.004307570431252166, 'mean_word_length': 4.4796527400976665, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4246866', 'n_tokens_mistral': 6003, 'n_tokens_neox': 5954, 'n_words': 1161}