<commit_before>/*
 * Licensed to the University of California, Berkeley under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package tachyon.underfs;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;

import javax.annotation.concurrent.ThreadSafe;

import com.google.common.base.Preconditions;

import tachyon.Constants;
import tachyon.TachyonURI;
import tachyon.collections.Pair;
import tachyon.conf.TachyonConf;
import tachyon.util.io.PathUtils;

/**
 * Tachyon stores data into an under layer file system. Any file system implementing this interface
 * can be a valid under layer file system
 */
@ThreadSafe
public abstract class UnderFileSystem {
  protected final TachyonConf mTachyonConf;

  /**
   * This variable indicates whether the underFS actually provides storage. Most UnderFS should
   * provide storage, but a dummyFS for example does not.
   */
  private boolean mProvidesStorage = true;

  /**
   * The different types of space indicate the total space, the free space and the space used in the
   * under file system.
   */
  public enum SpaceType {

    /**
     * Indicates the storage capacity of the under file system.
     */
    SPACE_TOTAL(0),

    /**
     * Indicates the amount of free space available in the under file system.
     */
    SPACE_FREE(1),

    /**
     * Indicates the amount of space used in the under file system.
     */
    SPACE_USED(2);

    private final int mValue;

    SpaceType(int value) {
      mValue = value;
    }

    /**
     * @return the integer value of this enum value
     */
    public int getValue() {
      return mValue;
    }
  }

  /**
   * Gets the UnderFileSystem instance according to its schema.
   *
   * @param path file path storing over the ufs
   * @param tachyonConf the {@link tachyon.conf.TachyonConf} instance
   * @return instance of the under layer file system
   */
  public static UnderFileSystem get(String path, TachyonConf tachyonConf) {
    return get(path, null, tachyonConf);
  }

  /**
   * Gets the UnderFileSystem instance according to its scheme and configuration.
   *
   * @param path file path storing over the ufs
   * @param ufsConf the configuration object for ufs only
   * @param tachyonConf the {@link tachyon.conf.TachyonConf} instance
   * @return instance of the under layer file system
   */
  public static UnderFileSystem get(String path, Object ufsConf, TachyonConf tachyonConf) {
    Preconditions.checkArgument(path != null, "path may not be null");
    Preconditions.checkNotNull(tachyonConf);

    // Use the registry to determine the factory to use to create the client
    return UnderFileSystemRegistry.create(path, tachyonConf, ufsConf);
  }

  /**
   * Type of under filesystem, to be used by {@link #getUnderFSType()} to determine which concrete
   * under filesystem implementation is being used. New types of under filesystem should be added
   * below and returned by the implementation of {@link #getUnderFSType()}.
   */
  public enum UnderFSType {
    LOCAL,
    HDFS,
    S3,
    GLUSTERFS,
    SWIFT,
    OSS,
  }

  /**
   * @return type of concrete under filesystem implementation
   */
  public abstract UnderFSType getUnderFSType();

  /**
   * Determines if the given path is on a Hadoop under file system
   *
   * To decide if a path should use the hadoop implementation, we check
   * {@link String#startsWith(String)} to see if the configured schemas are found.
   *
   * @param path the path in under filesystem
   * @param tachyonConf the configuration for Tachyon
   * @return true if the given path is on a Hadoop under file system, false otherwise
   */
  public static boolean isHadoopUnderFS(final String path, TachyonConf tachyonConf) {
    // TODO(hy): In Hadoop 2.x this can be replaced with the simpler call to
    // FileSystem.getFileSystemClass() without any need for having users explicitly declare the file
    // system schemes to treat as being HDFS. However as long as pre 2.x versions of Hadoop are
    // supported this is not an option and we have to continue to use this method.
    for (final String prefix : tachyonConf.getList(Constants.UNDERFS_HDFS_PREFIXS, ",")) {
      if (path.startsWith(prefix)) {
        return true;
      }
    }
    return false;
  }

  /**
   * Checks whether the underFS provides storage.
   *
   * @return true if the under filesystem provides storage, false otherwise
   */
  public boolean providesStorage() {
    return mProvidesStorage;
  }

  /**
   * Transform an input string like hdfs://host:port/dir, hdfs://host:port, file:///dir, /dir into a
   * pair of address and path. The returned pairs are ("hdfs://host:port", "/dir"),
   * ("hdfs://host:port", "/"), and ("/", "/dir"), respectively.
   *
   * @param path the input path string
   * @param tachyonConf the configuration for Tachyon
   * @return null if path does not start with tachyon://, tachyon-ft://, hdfs://, s3://, s3n://,
   *         file://, /. Or a pair of strings denoting the under FS address and the relative path
   *         relative to that address. For local FS (with prefixes file:// or /), the under FS
   *         address is "/" and the path starts with "/".
   */
  public static Pair<String, String> parse(TachyonURI path, TachyonConf tachyonConf) {
    Preconditions.checkNotNull(path);

    if (path.hasScheme()) {
      String header = path.getScheme() + "://";
      String authority = (path.hasAuthority()) ? path.getAuthority() : "";
      if (header.equals(Constants.HEADER) || header.equals(Constants.HEADER_FT)
          || isHadoopUnderFS(header, tachyonConf) || header.equals(Constants.HEADER_S3)
          || header.equals(Constants.HEADER_S3N) || header.equals(Constants.HEADER_OSS)) {
        if (path.getPath().isEmpty()) {
          return new Pair<String, String>(header + authority, TachyonURI.SEPARATOR);
        } else {
          return new Pair<String, String>(header + authority, path.getPath());
        }
      } else if (header.equals("file://")) {
        return new Pair<String, String>(TachyonURI.SEPARATOR, path.getPath());
      }
    } else if (path.isPathAbsolute()) {
      return new Pair<String, String>(TachyonURI.SEPARATOR, path.getPath());
    }

    return null;
  }

  protected UnderFileSystem(TachyonConf tachyonConf) {
    mTachyonConf = tachyonConf;
  }

  /**
   * Takes any necessary actions required to establish a connection to the under file system from
   * the given master host e.g. logging in
   * <p>
   * Depending on the implementation this may be a no-op
   * </p>
   *
   * @param conf Tachyon configuration
   * @param hostname The host that wants to connect to the under file system
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract void connectFromMaster(TachyonConf conf, String hostname) throws IOException;

  /**
   * Takes any necessary actions required to establish a connection to the under file system from
   * the given worker host e.g. logging in
   * <p>
   * Depending on the implementation this may be a no-op
   * </p>
   *
   * @param conf Tachyon configuration
   * @param hostname The host that wants to connect to the under file system
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract void connectFromWorker(TachyonConf conf, String hostname) throws IOException;

  /**
   * Closes this under file system.
   *
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract void close() throws IOException;

  /**
   * Creates a file in the under file system with the indicated name.
   *
   * @param path The file name
   * @return A {@code OutputStream} object
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract OutputStream create(String path) throws IOException;

  /**
   * Creates a file in the under file system with the indicated name and block size.
   *
   * @param path The file name
   * @param blockSizeByte The block size in bytes
   * @return A {@code OutputStream} object
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract OutputStream create(String path, int blockSizeByte) throws IOException;

  /**
   * Creates a file in the under file system with the indicated name, replication number and block
   * size.
   *
   * @param path The file name
   * @param replication The number of replications for each block
   * @param blockSizeByte The block size in bytes
   * @return A {@code OutputStream} object
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract OutputStream create(String path, short replication, int blockSizeByte)
      throws IOException;

  /**
   * Deletes a file or folder from the under file system with the indicated name.
   *
   * @param path The file or folder name
   * @param recursive Whether we delete folder and its children
   * @return true if succeed, false otherwise
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract boolean delete(String path, boolean recursive) throws IOException;

  /**
   * Checks if a file or folder exists in under file system.
   *
   * @param path The file name
   * @return true if succeed, false otherwise
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract boolean exists(String path) throws IOException;

  /**
   * Gets the block size of a file in under file system, in bytes.
   *
   * @param path The file name
   * @return file size in bytes
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract long getBlockSizeByte(String path) throws IOException;

  /**
   * Gets the configuration object for UnderFileSystem.
   *
   * @return configuration object used for concrete ufs instance
   */
  public abstract Object getConf();

  /**
   * Gets the list of locations of the indicated path.
   *
   * @param path The file name
   * @return The list of locations<commit_msg>[SMALLFIX] fix javadoc for getBlockSizeByte()<commit_after>/*
 * Licensed to the University of California, Berkeley under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package tachyon.underfs;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;

import javax.annotation.concurrent.ThreadSafe;

import com.google.common.base.Preconditions;

import tachyon.Constants;
import tachyon.TachyonURI;
import tachyon.collections.Pair;
import tachyon.conf.TachyonConf;
import tachyon.util.io.PathUtils;

/**
 * Tachyon stores data into an under layer file system. Any file system implementing this interface
 * can be a valid under layer file system
 */
@ThreadSafe
public abstract class UnderFileSystem {
  protected final TachyonConf mTachyonConf;

  /**
   * This variable indicates whether the underFS actually provides storage. Most UnderFS should
   * provide storage, but a dummyFS for example does not.
   */
  private boolean mProvidesStorage = true;

  /**
   * The different types of space indicate the total space, the free space and the space used in the
   * under file system.
   */
  public enum SpaceType {

    /**
     * Indicates the storage capacity of the under file system.
     */
    SPACE_TOTAL(0),

    /**
     * Indicates the amount of free space available in the under file system.
     */
    SPACE_FREE(1),

    /**
     * Indicates the amount of space used in the under file system.
     */
    SPACE_USED(2);

    private final int mValue;

    SpaceType(int value) {
      mValue = value;
    }

    /**
     * @return the integer value of this enum value
     */
    public int getValue() {
      return mValue;
    }
  }

  /**
   * Gets the UnderFileSystem instance according to its schema.
   *
   * @param path file path storing over the ufs
   * @param tachyonConf the {@link tachyon.conf.TachyonConf} instance
   * @return instance of the under layer file system
   */
  public static UnderFileSystem get(String path, TachyonConf tachyonConf) {
    return get(path, null, tachyonConf);
  }

  /**
   * Gets the UnderFileSystem instance according to its scheme and configuration.
   *
   * @param path file path storing over the ufs
   * @param ufsConf the configuration object for ufs only
   * @param tachyonConf the {@link tachyon.conf.TachyonConf} instance
   * @return instance of the under layer file system
   */
  public static UnderFileSystem get(String path, Object ufsConf, TachyonConf tachyonConf) {
    Preconditions.checkArgument(path != null, "path may not be null");
    Preconditions.checkNotNull(tachyonConf);

    // Use the registry to determine the factory to use to create the client
    return UnderFileSystemRegistry.create(path, tachyonConf, ufsConf);
  }

  /**
   * Type of under filesystem, to be used by {@link #getUnderFSType()} to determine which concrete
   * under filesystem implementation is being used. New types of under filesystem should be added
   * below and returned by the implementation of {@link #getUnderFSType()}.
   */
  public enum UnderFSType {
    LOCAL,
    HDFS,
    S3,
    GLUSTERFS,
    SWIFT,
    OSS,
  }

  /**
   * @return type of concrete under filesystem implementation
   */
  public abstract UnderFSType getUnderFSType();

  /**
   * Determines if the given path is on a Hadoop under file system
   *
   * To decide if a path should use the hadoop implementation, we check
   * {@link String#startsWith(String)} to see if the configured schemas are found.
   *
   * @param path the path in under filesystem
   * @param tachyonConf the configuration for Tachyon
   * @return true if the given path is on a Hadoop under file system, false otherwise
   */
  public static boolean isHadoopUnderFS(final String path, TachyonConf tachyonConf) {
    // TODO(hy): In Hadoop 2.x this can be replaced with the simpler call to
    // FileSystem.getFileSystemClass() without any need for having users explicitly declare the file
    // system schemes to treat as being HDFS. However as long as pre 2.x versions of Hadoop are
    // supported this is not an option and we have to continue to use this method.
    for (final String prefix : tachyonConf.getList(Constants.UNDERFS_HDFS_PREFIXS, ",")) {
      if (path.startsWith(prefix)) {
        return true;
      }
    }
    return false;
  }

  /**
   * Checks whether the underFS provides storage.
   *
   * @return true if the under filesystem provides storage, false otherwise
   */
  public boolean providesStorage() {
    return mProvidesStorage;
  }

  /**
   * Transform an input string like hdfs://host:port/dir, hdfs://host:port, file:///dir, /dir into a
   * pair of address and path. The returned pairs are ("hdfs://host:port", "/dir"),
   * ("hdfs://host:port", "/"), and ("/", "/dir"), respectively.
   *
   * @param path the input path string
   * @param tachyonConf the configuration for Tachyon
   * @return null if path does not start with tachyon://, tachyon-ft://, hdfs://, s3://, s3n://,
   *         file://, /. Or a pair of strings denoting the under FS address and the relative path
   *         relative to that address. For local FS (with prefixes file:// or /), the under FS
   *         address is "/" and the path starts with "/".
   */
  public static Pair<String, String> parse(TachyonURI path, TachyonConf tachyonConf) {
    Preconditions.checkNotNull(path);

    if (path.hasScheme()) {
      String header = path.getScheme() + "://";
      String authority = (path.hasAuthority()) ? path.getAuthority() : "";
      if (header.equals(Constants.HEADER) || header.equals(Constants.HEADER_FT)
          || isHadoopUnderFS(header, tachyonConf) || header.equals(Constants.HEADER_S3)
          || header.equals(Constants.HEADER_S3N) || header.equals(Constants.HEADER_OSS)) {
        if (path.getPath().isEmpty()) {
          return new Pair<String, String>(header + authority, TachyonURI.SEPARATOR);
        } else {
          return new Pair<String, String>(header + authority, path.getPath());
        }
      } else if (header.equals("file://")) {
        return new Pair<String, String>(TachyonURI.SEPARATOR, path.getPath());
      }
    } else if (path.isPathAbsolute()) {
      return new Pair<String, String>(TachyonURI.SEPARATOR, path.getPath());
    }

    return null;
  }

  protected UnderFileSystem(TachyonConf tachyonConf) {
    mTachyonConf = tachyonConf;
  }

  /**
   * Takes any necessary actions required to establish a connection to the under file system from
   * the given master host e.g. logging in
   * <p>
   * Depending on the implementation this may be a no-op
   * </p>
   *
   * @param conf Tachyon configuration
   * @param hostname The host that wants to connect to the under file system
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract void connectFromMaster(TachyonConf conf, String hostname) throws IOException;

  /**
   * Takes any necessary actions required to establish a connection to the under file system from
   * the given worker host e.g. logging in
   * <p>
   * Depending on the implementation this may be a no-op
   * </p>
   *
   * @param conf Tachyon configuration
   * @param hostname The host that wants to connect to the under file system
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract void connectFromWorker(TachyonConf conf, String hostname) throws IOException;

  /**
   * Closes this under file system.
   *
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract void close() throws IOException;

  /**
   * Creates a file in the under file system with the indicated name.
   *
   * @param path The file name
   * @return A {@code OutputStream} object
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract OutputStream create(String path) throws IOException;

  /**
   * Creates a file in the under file system with the indicated name and block size.
   *
   * @param path The file name
   * @param blockSizeByte The block size in bytes
   * @return A {@code OutputStream} object
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract OutputStream create(String path, int blockSizeByte) throws IOException;

  /**
   * Creates a file in the under file system with the indicated name, replication number and block
   * size.
   *
   * @param path The file name
   * @param replication The number of replications for each block
   * @param blockSizeByte The block size in bytes
   * @return A {@code OutputStream} object
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract OutputStream create(String path, short replication, int blockSizeByte)
      throws IOException;

  /**
   * Deletes a file or folder from the under file system with the indicated name.
   *
   * @param path The file or folder name
   * @param recursive Whether we delete folder and its children
   * @return true if succeed, false otherwise
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract boolean delete(String path, boolean recursive) throws IOException;

  /**
   * Checks if a file or folder exists in under file system.
   *
   * @param path The file name
   * @return true if succeed, false otherwise
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract boolean exists(String path) throws IOException;

  /**
   * Gets the block size of a file in under file system, in bytes.
   *
   * @param path The file name
   * @return the block size in bytes
   * @throws IOException if a non-Tachyon error occurs
   */
  public abstract long getBlockSizeByte(String path) throws IOException;

  /**
   * Gets the configuration object for UnderFileSystem.
   *
   * @return configuration object used for concrete ufs instance
   */
  public abstract Object getConf();

  /**
   * Gets the list of locations of the indicated path.
   *
   * @param path The file name
   * @return The list of locations
{'fraction_non_alphanumeric': 0.08294026992948209, 'fraction_numerical': 0.0013076168682576005, 'mean_word_length': 3.1775263363246196, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2765587', 'n_tokens_mistral': 6097, 'n_tokens_neox': 5839, 'n_words': 2440}