<commit_before>/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.hadoop;

import alluxio.CommonTestUtils;
import alluxio.Configuration;
import alluxio.ConfigurationTestUtils;
import alluxio.Constants;
import alluxio.client.ClientContext;
import alluxio.client.block.BlockStoreContext;
import alluxio.client.file.FileSystemContext;
import alluxio.client.file.FileSystemMasterClient;
import alluxio.client.lineage.LineageContext;
import alluxio.client.util.ClientTestUtils;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.MockClassLoader;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.powermock.reflect.Whitebox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * Unit tests for {@link FileSystem}.
 */
@RunWith(PowerMockRunner.class)
@PrepareForTest({FileSystemContext.class, FileSystemMasterClient.class, UserGroupInformation.class})
/*
 * [ALLUXIO-1384] Tell PowerMock to defer the loading of javax.security classes to the system
 * classloader in order to avoid linkage error when running this test with CDH.
 * See https://code.google.com/p/powermock/wiki/FAQ.
 */
@PowerMockIgnore("javax.security.*")
/**
 * Tests for {@link AbstractFileSystem}.
 */
public class AbstractFileSystemTest {
  private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);
  private FileSystemContext mMockFileSystemContext;

  /**
   * Sets up the configuration before a test runs.
   */
  @Before
  public void before() throws Exception {
    mockUserGroupInformation();
    mockMasterClient();

    if (isHadoop1x()) {
      LOG.debug("Running Alluxio FS tests against hadoop 1x");
    } else if (isHadoop2x()) {
      LOG.debug("Running Alluxio FS tests against hadoop 2x");
    } else {
      LOG.warn("Running Alluxio FS tests against untargeted Hadoop version: " + getHadoopVersion());
    }
  }

  @After
  public void after() {
    ConfigurationTestUtils.resetConfiguration();
    ClientTestUtils.resetClient();
  }

  private ClassLoader getClassLoader(Class<?> clazz) {
    // Power Mock makes this hard, so try to hack it
    ClassLoader cl = clazz.getClassLoader();
    if (cl instanceof MockClassLoader) {
      cl = cl.getParent();
    }
    return cl;
  }

  private String getHadoopVersion() {
    try {
      final URL url = getSourcePath(org.apache.hadoop.fs.FileSystem.class);
      final File path = new File(url.toURI());
      final String[] splits = path.getName().split("-");
      final String last = splits[splits.length - 1];
      return last.substring(0, last.lastIndexOf("."));
    } catch (URISyntaxException e) {
      throw new AssertionError(e);
    }
  }

  private URL getSourcePath(Class<?> clazz) {
    try {
      clazz = getClassLoader(clazz).loadClass(clazz.getName());
      return clazz.getProtectionDomain().getCodeSource().getLocation();
    } catch (ClassNotFoundException e) {
      throw new AssertionError("Unable to find class " + clazz.getName());
    }
  }

  /**
   * Ensures that Hadoop loads {@link FaultTolerantFileSystem} when configured.
   */
  @Test
  public void hadoopShouldLoadFaultTolerantFileSystemWhenConfiguredTest() throws Exception {
    final org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
    if (isHadoop1x()) {
      conf.set("fs." + Constants.SCHEME_FT + ".impl", FaultTolerantFileSystem.class.getName());
    }

    // when
    final URI uri = URI.create(Constants.HEADER_FT + "localhost:19998/tmp/path.txt");

    Configuration.set(Constants.MASTER_HOSTNAME, uri.getHost());
    Configuration.set(Constants.MASTER_RPC_PORT, Integer.toString(uri.getPort()));
    Configuration.set(Constants.ZOOKEEPER_ENABLED, "true");

    final org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, conf);

    Assert.assertTrue(fs instanceof FaultTolerantFileSystem);

    PowerMockito.verifyStatic();
    alluxio.client.file.FileSystem.Factory.get();
    ClientTestUtils.resetClient();
  }

  /**
   * Ensures that Hadoop loads the Alluxio file system when configured.
   */
  @Test
  public void hadoopShouldLoadFileSystemWhenConfiguredTest() throws Exception {
    final org.apache.hadoop.conf.Configuration conf = getConf();

    // when
    final URI uri = URI.create(Constants.HEADER + "localhost:19998/tmp/path.txt");

    Configuration.set(Constants.MASTER_HOSTNAME, uri.getHost());
    Configuration.set(Constants.MASTER_RPC_PORT, Integer.toString(uri.getPort()));
    Configuration.set(Constants.ZOOKEEPER_ENABLED, "false");

    final org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, conf);

    Assert.assertTrue(fs instanceof FileSystem);

    PowerMockito.verifyStatic();
    alluxio.client.file.FileSystem.Factory.get();
    ClientTestUtils.resetClient();
  }

  /**
  * Ensures that FileNotFoundException is thrown when listStatus is called on a nonexistent path.
  */
  @Test
  public void throwFileNotFoundExceptionWhenListStatusNonExistingTest() throws Exception {
    final org.apache.hadoop.conf.Configuration conf = getConf();
    conf.set(Constants.MASTER_HOSTNAME, "localhost");
    conf.set(Constants.MASTER_RPC_PORT, "19998");
    ClientContext.init();
    org.apache.hadoop.fs.FileSystem fs = FileSystem.get(conf);
    StringBuilder path = new StringBuilder("/ALLUXIO-2036.");
    path.append(System.currentTimeMillis()).append(".txt");
    try {
      fs.listStatus(new Path(path.toString()));
      Assert.fail("Listing the status of a nonexistent file should throw an exception");
    } catch (FileNotFoundException e) {
      // This exception is expected
    } finally {
      fs.close();
    }
  }

  /**
   * Tests that initializing the {@link AbstractFileSystem} will reinitialize contexts to pick up
   * changes to the master address.
   */
  @Test
  public void resetContextTest() throws Exception {
    // Change to otherhost:410
    URI uri = URI.create(Constants.HEADER + "otherhost:410/");
    org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, getConf());

    // Make sure all contexts are using the new address
    InetSocketAddress newAddress = new InetSocketAddress("otherhost", 410);
    Assert.assertEquals(newAddress, ClientContext.getMasterAddress());
    Assert.assertEquals(newAddress, CommonTestUtils.getInternalState(BlockStoreContext.INSTANCE,
        "mBlockMasterClientPool", "mMasterAddress"));
    // Once from calling FileSystem.get
    Mockito.verify(mMockFileSystemContext).reset();
    Assert.assertEquals(newAddress, CommonTestUtils.getInternalState(LineageContext.INSTANCE,
        "mLineageMasterClientPool", "mMasterAddress"));<commit_msg>Revert "fix ALLUXIO-2036"<commit_after>/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.hadoop;

import alluxio.CommonTestUtils;
import alluxio.Configuration;
import alluxio.ConfigurationTestUtils;
import alluxio.Constants;
import alluxio.client.ClientContext;
import alluxio.client.block.BlockStoreContext;
import alluxio.client.file.FileSystemContext;
import alluxio.client.file.FileSystemMasterClient;
import alluxio.client.lineage.LineageContext;
import alluxio.client.util.ClientTestUtils;

import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.MockClassLoader;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.powermock.reflect.Whitebox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * Unit tests for {@link FileSystem}.
 */
@RunWith(PowerMockRunner.class)
@PrepareForTest({FileSystemContext.class, FileSystemMasterClient.class, UserGroupInformation.class})
/*
 * [ALLUXIO-1384] Tell PowerMock to defer the loading of javax.security classes to the system
 * classloader in order to avoid linkage error when running this test with CDH.
 * See https://code.google.com/p/powermock/wiki/FAQ.
 */
@PowerMockIgnore("javax.security.*")
/**
 * Tests for {@link AbstractFileSystem}.
 */
public class AbstractFileSystemTest {
  private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);
  private FileSystemContext mMockFileSystemContext;

  /**
   * Sets up the configuration before a test runs.
   */
  @Before
  public void before() throws Exception {
    mockUserGroupInformation();
    mockMasterClient();

    if (isHadoop1x()) {
      LOG.debug("Running Alluxio FS tests against hadoop 1x");
    } else if (isHadoop2x()) {
      LOG.debug("Running Alluxio FS tests against hadoop 2x");
    } else {
      LOG.warn("Running Alluxio FS tests against untargeted Hadoop version: " + getHadoopVersion());
    }
  }

  @After
  public void after() {
    ConfigurationTestUtils.resetConfiguration();
    ClientTestUtils.resetClient();
  }

  private ClassLoader getClassLoader(Class<?> clazz) {
    // Power Mock makes this hard, so try to hack it
    ClassLoader cl = clazz.getClassLoader();
    if (cl instanceof MockClassLoader) {
      cl = cl.getParent();
    }
    return cl;
  }

  private String getHadoopVersion() {
    try {
      final URL url = getSourcePath(org.apache.hadoop.fs.FileSystem.class);
      final File path = new File(url.toURI());
      final String[] splits = path.getName().split("-");
      final String last = splits[splits.length - 1];
      return last.substring(0, last.lastIndexOf("."));
    } catch (URISyntaxException e) {
      throw new AssertionError(e);
    }
  }

  private URL getSourcePath(Class<?> clazz) {
    try {
      clazz = getClassLoader(clazz).loadClass(clazz.getName());
      return clazz.getProtectionDomain().getCodeSource().getLocation();
    } catch (ClassNotFoundException e) {
      throw new AssertionError("Unable to find class " + clazz.getName());
    }
  }

  /**
   * Ensures that Hadoop loads {@link FaultTolerantFileSystem} when configured.
   */
  @Test
  public void hadoopShouldLoadFaultTolerantFileSystemWhenConfiguredTest() throws Exception {
    final org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
    if (isHadoop1x()) {
      conf.set("fs." + Constants.SCHEME_FT + ".impl", FaultTolerantFileSystem.class.getName());
    }

    // when
    final URI uri = URI.create(Constants.HEADER_FT + "localhost:19998/tmp/path.txt");

    Configuration.set(Constants.MASTER_HOSTNAME, uri.getHost());
    Configuration.set(Constants.MASTER_RPC_PORT, Integer.toString(uri.getPort()));
    Configuration.set(Constants.ZOOKEEPER_ENABLED, "true");

    final org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, conf);

    Assert.assertTrue(fs instanceof FaultTolerantFileSystem);

    PowerMockito.verifyStatic();
    alluxio.client.file.FileSystem.Factory.get();
    ClientTestUtils.resetClient();
  }

  /**
   * Ensures that Hadoop loads the Alluxio file system when configured.
   */
  @Test
  public void hadoopShouldLoadFileSystemWhenConfiguredTest() throws Exception {
    final org.apache.hadoop.conf.Configuration conf = getConf();

    // when
    final URI uri = URI.create(Constants.HEADER + "localhost:19998/tmp/path.txt");

    Configuration.set(Constants.MASTER_HOSTNAME, uri.getHost());
    Configuration.set(Constants.MASTER_RPC_PORT, Integer.toString(uri.getPort()));
    Configuration.set(Constants.ZOOKEEPER_ENABLED, "false");

    final org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, conf);

    Assert.assertTrue(fs instanceof FileSystem);

    PowerMockito.verifyStatic();
    alluxio.client.file.FileSystem.Factory.get();
    ClientTestUtils.resetClient();
  }

  /**
   * Tests that initializing the {@link AbstractFileSystem} will reinitialize contexts to pick up
   * changes to the master address.
   */
  @Test
  public void resetContextTest() throws Exception {
    // Change to otherhost:410
    URI uri = URI.create(Constants.HEADER + "otherhost:410/");
    org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, getConf());

    // Make sure all contexts are using the new address
    InetSocketAddress newAddress = new InetSocketAddress("otherhost", 410);
    Assert.assertEquals(newAddress, ClientContext.getMasterAddress());
    Assert.assertEquals(newAddress, CommonTestUtils.getInternalState(BlockStoreContext.INSTANCE,
        "mBlockMasterClientPool", "mMasterAddress"));
    // Once from calling FileSystem.get
    Mockito.verify(mMockFileSystemContext).reset();
    Assert.assertEquals(newAddress, CommonTestUtils.getInternalState(LineageContext.INSTANCE,
        "mLineageMasterClientPool", "mMasterAddress"));
{'fraction_non_alphanumeric': 0.10979775895053294, 'fraction_numerical': 0.005944247062038809, 'mean_word_length': 4.282208588957055, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6648095', 'n_tokens_mistral': 4351, 'n_tokens_neox': 4247, 'n_words': 1016}