<commit_before>/*
 * Copyright 2015 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

/// <reference path="../../../../../../externs/ts/node/node.d.ts" />
/// <reference path="../../../../../../externs/ts/node/gcloud.d.ts" />
/// <reference path="../common/interfaces.d.ts" />
import gcloud = require('gcloud');
import pathlib = require('path');


/**
 * Manages storage operations backed by Google Cloud Storage (GCS).
 */
export class GoogleCloudStorage implements app.IStorage {

  _bucket: GCloud.Bucket;

  /**
   * Constructor.
   *
   * @param bucket The gcloud bucket client to use for accessing storage.
   */
  constructor(bucket: GCloud.Bucket) {
    this._bucket = bucket;
  }

  /**
   * Asynchronously deletes the GCS object at the given path.
   *
   * @param path The file system path to write to, relative to the root path.
   * @param callback Callback to invoke upon completion of the write operation.
   */
  delete(path: string, callback: app.Callback<void>) {
    var file = this._bucket.file(path);
    file.delete((error) => {

      if (error) {
        callback(error);
        return;
      }

      callback(null);
    });
  }

  /**
   * Asynchronously enumerates the resources that match the given directory path.
   *
   * @param directoryPath The storage path for which to enumerate resources.
   * @param recursive Should the listing operation recursively enumerate sub-directories?
   * @param callback Completion callback to invoke.
   */
  list(directoryPath: string, recursive: boolean, callback: app.Callback<app.Resource[]>) {
    var query: GCloud.Query = {
      prefix: this._toGcsPath(directoryPath)
    };


    this._bucket.getFiles(query, (error, files, nextPageToken) => {
      if (error) {
        callback(error);
        return;
      }

      // TODO(bryantd): add support for paging within the list API eventually via nextQuery GCS
      // token. For now, truncate the listing response to the first page of results from GCS to
      // avoid returning a response with unbounded size.

      // Get the paths to all objects/directories within that matched the query.
      var resources = files.map(file => this._toResource(file.name));

      // Filter the resources to only those files and directories directly contained within the
      // query path/directory if a non-recursive listing was requested.
      resources = this._selectWithinDirectory(directoryPath, resources, recursive);

      callback(null, this._selectNotebooks(resources));
    });
  }

  /**
   * Asynchronously moves the resource from source path to destination path.
   *
   * Note: this operation only supports moving individual objects, not sets of objects
   * that share some common prefix ("directories").
   *
   * @param sourcePath The path to be moved.
   * @param destinationPath The path to move to.
   * @param callback Completion callback.
   */
  move(sourcePath: string, destinationPath: string, callback: app.Callback<void>) {
    var source = this._bucket.file(sourcePath);
    source.copy(destinationPath, (error) => {
      if (error) {
        callback(error);
        return;
      }

      // Successfully copied to the destination; Now delete the source.
      this.delete(sourcePath, (error) => {
        if (error) {
          callback(error);
          return;
        }

        callback(null);
      });
    });
  }

  /**
   * Asynchronously opens and reads from the GCS object at the given path.
   *
   * @param path The storage path to read.
   * @param callback Callback to invoke upon completion of the read operation.
   */
  read(path: string, callback: app.Callback<string>) {
    var file = this._bucket.file(path);

    file.download((error, buffer) => {
      if (error) {
        // An error reason of notFound indicates that the specified read failed because the object
        // doesn't exist. Invoke the callback with a null data value (but no error) to signal this.
        if (error.reason == 'notFound') {
          callback(null, null);
          return;
        } else {
          // Any other errors are passed back to caller to handle.
          callback(error);
          return;
        }
      }

      // File content is returned as a buffer object, so deserialize it to utf-8.
      var data = buffer.toString('utf8');
      callback(null, data);
    });
  }

  /**
   * Asynchronously writes the given data string to the GCS object referenced by the given path.
   *
   * @param path The file system path to write to, relative to the root path.
   * @param data The data string to write.
   * @param callback Callback to invoke upon completion of the write operation.
   */
  write(path: string, data: string, callback: app.Callback<void>) {
    var file = this._bucket.file(path);

    file.createWriteStream().end(data, 'utf8', (error: Error) => {
      if (error) {
        callback(error);
        return;
      }

      callback(null);
    });
  }

  /**
   * Detects if the given GCS path represents a "directory".
   *
   * @param gcsPath The GCS resource path to check.
   * @return Boolean to indicate if the given resource path represents a GCS "directory".
   */
  _isGcsDirectory(gcsPath: string): boolean {
    // GCS denotes "directories" as object paths with a trailing slash.
    return gcsPath[gcsPath.length - 1] == '/';
  }

  /**
   * Selects directories and notebook files from the specified list of resources.
   *<commit_msg>Fixes read and write operations to translate storage to gcs paths<commit_after>/*
 * Copyright 2015 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

/// <reference path="../../../../../../externs/ts/node/node.d.ts" />
/// <reference path="../../../../../../externs/ts/node/gcloud.d.ts" />
/// <reference path="../common/interfaces.d.ts" />
import gcloud = require('gcloud');
import pathlib = require('path');


/**
 * Manages storage operations backed by Google Cloud Storage (GCS).
 */
export class GoogleCloudStorage implements app.IStorage {

  _bucket: GCloud.Bucket;

  /**
   * Constructor.
   *
   * @param bucket The gcloud bucket client to use for accessing storage.
   */
  constructor(bucket: GCloud.Bucket) {
    this._bucket = bucket;
  }

  /**
   * Asynchronously deletes the GCS object at the given path.
   *
   * @param path The file system path to write to, relative to the root path.
   * @param callback Callback to invoke upon completion of the write operation.
   */
  delete(path: string, callback: app.Callback<void>) {
    var file = this._bucket.file(path);
    file.delete((error) => {

      if (error) {
        callback(error);
        return;
      }

      callback(null);
    });
  }

  /**
   * Asynchronously enumerates the resources that match the given directory path.
   *
   * @param directoryPath The storage path for which to enumerate resources.
   * @param recursive Should the listing operation recursively enumerate sub-directories?
   * @param callback Completion callback to invoke.
   */
  list(directoryPath: string, recursive: boolean, callback: app.Callback<app.Resource[]>) {
    var query: GCloud.Query = {
      prefix: this._toGcsPath(directoryPath)
    };


    this._bucket.getFiles(query, (error, files, nextPageToken) => {
      if (error) {
        callback(error);
        return;
      }

      // TODO(bryantd): add support for paging within the list API eventually via nextQuery GCS
      // token. For now, truncate the listing response to the first page of results from GCS to
      // avoid returning a response with unbounded size.

      // Get the paths to all objects/directories within that matched the query.
      var resources = files.map(file => this._toResource(file.name));

      // Filter the resources to only those files and directories directly contained within the
      // query path/directory if a non-recursive listing was requested.
      resources = this._selectWithinDirectory(directoryPath, resources, recursive);

      callback(null, this._selectNotebooks(resources));
    });
  }

  /**
   * Asynchronously moves the resource from source path to destination path.
   *
   * Note: this operation only supports moving individual objects, not sets of objects
   * that share some common prefix ("directories").
   *
   * @param sourcePath The path to be moved.
   * @param destinationPath The path to move to.
   * @param callback Completion callback.
   */
  move(sourcePath: string, destinationPath: string, callback: app.Callback<void>) {
    var source = this._bucket.file(sourcePath);
    source.copy(destinationPath, (error) => {
      if (error) {
        callback(error);
        return;
      }

      // Successfully copied to the destination; Now delete the source.
      this.delete(sourcePath, (error) => {
        if (error) {
          callback(error);
          return;
        }

        callback(null);
      });
    });
  }

  /**
   * Asynchronously opens and reads from the GCS object at the given path.
   *
   * @param path The storage path to read.
   * @param callback Callback to invoke upon completion of the read operation.
   */
  read(path: string, callback: app.Callback<string>) {
    var file = this._bucket.file(this._toGcsPath(path));

    file.download((error, buffer) => {
      if (error) {
        // An error reason of notFound indicates that the specified read failed because the object
        // doesn't exist. Invoke the callback with a null data value (but no error) to signal this.
        if (error.reason == 'notFound') {
          callback(null, null);
          return;
        } else {
          // Any other errors are passed back to caller to handle.
          callback(error);
          return;
        }
      }

      // File content is returned as a buffer object, so deserialize it to utf-8.
      var data = buffer.toString('utf8');
      callback(null, data);
    });
  }

  /**
   * Asynchronously writes the given data string to the GCS object referenced by the given path.
   *
   * @param path The file system path to write to, relative to the root path.
   * @param data The data string to write.
   * @param callback Callback to invoke upon completion of the write operation.
   */
  write(path: string, data: string, callback: app.Callback<void>) {
    var file = this._bucket.file(this._toGcsPath(path));

    file.createWriteStream().end(data, 'utf8', (error: Error) => {
      if (error) {
        callback(error);
        return;
      }

      callback(null);
    });
  }

  /**
   * Detects if the given GCS path represents a "directory".
   *
   * @param gcsPath The GCS resource path to check.
   * @return Boolean to indicate if the given resource path represents a GCS "directory".
   */
  _isGcsDirectory(gcsPath: string): boolean {
    // GCS denotes "directories" as object paths with a trailing slash.
    return gcsPath[gcsPath.length - 1] == '/';
  }

  /**
   * Selects directories and notebook files from the specified list of resources.
   *
{'fraction_non_alphanumeric': 0.10477299185098952, 'fraction_numerical': 0.0019956760352569432, 'mean_word_length': 2.7017543859649122, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4812364', 'n_tokens_mistral': 3295, 'n_tokens_neox': 3190, 'n_words': 1242}