<commit_before>/*
 * Licensed to the University of California, Berkeley under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package tachyon.master;

import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import junit.framework.Assert;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import tachyon.Constants;
import tachyon.TachyonURI;
import tachyon.TestUtils;
import tachyon.UnderFileSystem;
import tachyon.client.TachyonFile;
import tachyon.client.TachyonFS;
import tachyon.client.WriteType;
import tachyon.conf.MasterConf;
import tachyon.thrift.ClientFileInfo;
import tachyon.thrift.FileDoesNotExistException;
import tachyon.thrift.InvalidPathException;

/**
 * Test master journal, including image and edit log. Most tests will test edit log first, followed
 * by the image.
 */
public class JournalTest {
  private LocalTachyonCluster mLocalTachyonCluster = null;
  private TachyonFS mTfs = null;
  private TachyonURI mRootUri = new TachyonURI(TachyonURI.SEPARATOR);
  private final ExecutorService mExecutorService = Executors.newFixedThreadPool(2);

  /**
   * Test add block
   * 
   * @throws Exception
   */
  @Test
  public void AddBlockTest() throws Exception {
    TachyonURI uri = new TachyonURI("/xyz");
    mTfs.createFile(uri, 64);
    TachyonFile file = mTfs.getFile(uri);
    OutputStream os = file.getOutStream(WriteType.MUST_CACHE);
    for (int k = 0; k < 1000; k ++) {
      os.write(k);
    }
    os.close();
    ClientFileInfo fInfo = mLocalTachyonCluster.getMasterInfo().getClientFileInfo(uri);
    mLocalTachyonCluster.stopTFS();
    AddBlockTestUtil(fInfo);
    String editLogPath = mLocalTachyonCluster.getEditLogPath();
    UnderFileSystem.get(editLogPath).delete(editLogPath, true);
    AddBlockTestUtil(fInfo);
  }

  private void AddBlockTestUtil(ClientFileInfo fileInfo) throws IOException, InvalidPathException,
      FileDoesNotExistException {
    Journal journal = new Journal(MasterConf.get().JOURNAL_FOLDER, "image.data", "log.data");
    MasterInfo info = new MasterInfo(new InetSocketAddress(9999), journal, mExecutorService);
    info.init();
    Assert.assertEquals(2, info.ls(mRootUri, true).size());
    Assert.assertTrue(info.getFileId(mRootUri) != -1);
    Assert.assertTrue(info.getFileId(new TachyonURI("/xyz")) != -1);
    int temp = fileInfo.inMemoryPercentage;
    fileInfo.setInMemoryPercentage(0);
    Assert.assertEquals(fileInfo, info.getClientFileInfo(info.getFileId(new TachyonURI("/xyz"))));
    fileInfo.setInMemoryPercentage(temp);
    info.stop();
  }

  /**
   * Test add checkpoint
   * 
   * @throws Exception
   */
  @Test
  public void AddCheckpointTest() throws Exception {
    TestUtils.createByteFile(mTfs, "/xyz", WriteType.THROUGH, 10);
    ClientFileInfo fInfo =
        mLocalTachyonCluster.getMasterInfo().getClientFileInfo(new TachyonURI("/xyz"));
    mTfs.createFile(new TachyonURI("/xyz_ck"), new TachyonURI(fInfo.getUfsPath()));
    ClientFileInfo ckFileInfo =
        mLocalTachyonCluster.getMasterInfo().getClientFileInfo(new TachyonURI("/xyz_ck"));
    mLocalTachyonCluster.stopTFS();
    AddCheckpointTestUtil(fInfo, ckFileInfo);
    String editLogPath = mLocalTachyonCluster.getEditLogPath();
    UnderFileSystem.get(editLogPath).delete(editLogPath, true);
    AddCheckpointTestUtil(fInfo, ckFileInfo);
  }

  private void AddCheckpointTestUtil(ClientFileInfo fileInfo, ClientFileInfo ckFileInfo)
      throws IOException, InvalidPathException, FileDoesNotExistException {
    Journal journal = new Journal(MasterConf.get().JOURNAL_FOLDER, "image.data", "log.data");
    MasterInfo info = new MasterInfo(new InetSocketAddress(9999), journal, mExecutorService);
    info.init();
    Assert.assertEquals(3, info.ls(mRootUri, true).size());
    Assert.assertTrue(info.getFileId(mRootUri) != -1);
    Assert.assertTrue(info.getFileId(new TachyonURI("/xyz")) != -1);
    Assert.assertTrue(info.getFileId(new TachyonURI("/xyz_ck")) != -1);
    Assert.assertEquals(
        fileInfo, info.getClientFileInfo(info.getFileId(new TachyonURI("/xyz"))));
    Assert.assertEquals(
        ckFileInfo, info.getClientFileInfo(info.getFileId(new TachyonURI("/xyz_ck"))));
    info.stop();
  }

  /**
   * mLocalTachyonCluster is not closed in after(). Need to be closed by any test method.
   * 
   * @throws Exception
   */
  @After
  public final void after() throws Exception {
    mLocalTachyonCluster.stop();
    mExecutorService.shutdown();
    System.clearProperty("tachyon.user.quota.unit.bytes");
    System.clearProperty("fs.hdfs.impl.disable.cache");
  }

  @Before
  public final void before() throws IOException {
    System.setProperty("tachyon.user.quota.unit.bytes", "1000");
    System.setProperty("fs.hdfs.impl.disable.cache", "true");
    mLocalTachyonCluster = new LocalTachyonCluster(10000);
    mLocalTachyonCluster.start();
    mTfs = mLocalTachyonCluster.getClient();
  }

  /**
   * Test completed Editlog deletion
   * 
   * @throws Exception
   */
  @Test
  public void CompltedEditLogDeletionTest() throws Exception {
    Journal journal = mLocalTachyonCluster.getMasterInfo().getJournal();
    journal.setMaxLogSize(Constants.KB);
    for (int i = 0; i < 124; i ++) {
      mTfs.createFile(new TachyonURI("/a" + i), (i + 10) / 10 * 64);
    }
    mLocalTachyonCluster.stopTFS();
    String editLogPath = mLocalTachyonCluster.getEditLogPath();
    String completedPath =
        editLogPath.substring(0, editLogPath.lastIndexOf(TachyonURI.SEPARATOR)) + "/completed";
    Assert.assertTrue(UnderFileSystem.get(completedPath).list(completedPath).length > 1);
    MultiEditLogTestUtil();<commit_msg>Simple typo fix of the JournalTest.CompletedEditLogDeletionTest method name.<commit_after>/*
 * Licensed to the University of California, Berkeley under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package tachyon.master;

import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import junit.framework.Assert;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import tachyon.Constants;
import tachyon.TachyonURI;
import tachyon.TestUtils;
import tachyon.UnderFileSystem;
import tachyon.client.TachyonFile;
import tachyon.client.TachyonFS;
import tachyon.client.WriteType;
import tachyon.conf.MasterConf;
import tachyon.thrift.ClientFileInfo;
import tachyon.thrift.FileDoesNotExistException;
import tachyon.thrift.InvalidPathException;

/**
 * Test master journal, including image and edit log. Most tests will test edit log first, followed
 * by the image.
 */
public class JournalTest {
  private LocalTachyonCluster mLocalTachyonCluster = null;
  private TachyonFS mTfs = null;
  private TachyonURI mRootUri = new TachyonURI(TachyonURI.SEPARATOR);
  private final ExecutorService mExecutorService = Executors.newFixedThreadPool(2);

  /**
   * Test add block
   * 
   * @throws Exception
   */
  @Test
  public void AddBlockTest() throws Exception {
    TachyonURI uri = new TachyonURI("/xyz");
    mTfs.createFile(uri, 64);
    TachyonFile file = mTfs.getFile(uri);
    OutputStream os = file.getOutStream(WriteType.MUST_CACHE);
    for (int k = 0; k < 1000; k ++) {
      os.write(k);
    }
    os.close();
    ClientFileInfo fInfo = mLocalTachyonCluster.getMasterInfo().getClientFileInfo(uri);
    mLocalTachyonCluster.stopTFS();
    AddBlockTestUtil(fInfo);
    String editLogPath = mLocalTachyonCluster.getEditLogPath();
    UnderFileSystem.get(editLogPath).delete(editLogPath, true);
    AddBlockTestUtil(fInfo);
  }

  private void AddBlockTestUtil(ClientFileInfo fileInfo) throws IOException, InvalidPathException,
      FileDoesNotExistException {
    Journal journal = new Journal(MasterConf.get().JOURNAL_FOLDER, "image.data", "log.data");
    MasterInfo info = new MasterInfo(new InetSocketAddress(9999), journal, mExecutorService);
    info.init();
    Assert.assertEquals(2, info.ls(mRootUri, true).size());
    Assert.assertTrue(info.getFileId(mRootUri) != -1);
    Assert.assertTrue(info.getFileId(new TachyonURI("/xyz")) != -1);
    int temp = fileInfo.inMemoryPercentage;
    fileInfo.setInMemoryPercentage(0);
    Assert.assertEquals(fileInfo, info.getClientFileInfo(info.getFileId(new TachyonURI("/xyz"))));
    fileInfo.setInMemoryPercentage(temp);
    info.stop();
  }

  /**
   * Test add checkpoint
   * 
   * @throws Exception
   */
  @Test
  public void AddCheckpointTest() throws Exception {
    TestUtils.createByteFile(mTfs, "/xyz", WriteType.THROUGH, 10);
    ClientFileInfo fInfo =
        mLocalTachyonCluster.getMasterInfo().getClientFileInfo(new TachyonURI("/xyz"));
    mTfs.createFile(new TachyonURI("/xyz_ck"), new TachyonURI(fInfo.getUfsPath()));
    ClientFileInfo ckFileInfo =
        mLocalTachyonCluster.getMasterInfo().getClientFileInfo(new TachyonURI("/xyz_ck"));
    mLocalTachyonCluster.stopTFS();
    AddCheckpointTestUtil(fInfo, ckFileInfo);
    String editLogPath = mLocalTachyonCluster.getEditLogPath();
    UnderFileSystem.get(editLogPath).delete(editLogPath, true);
    AddCheckpointTestUtil(fInfo, ckFileInfo);
  }

  private void AddCheckpointTestUtil(ClientFileInfo fileInfo, ClientFileInfo ckFileInfo)
      throws IOException, InvalidPathException, FileDoesNotExistException {
    Journal journal = new Journal(MasterConf.get().JOURNAL_FOLDER, "image.data", "log.data");
    MasterInfo info = new MasterInfo(new InetSocketAddress(9999), journal, mExecutorService);
    info.init();
    Assert.assertEquals(3, info.ls(mRootUri, true).size());
    Assert.assertTrue(info.getFileId(mRootUri) != -1);
    Assert.assertTrue(info.getFileId(new TachyonURI("/xyz")) != -1);
    Assert.assertTrue(info.getFileId(new TachyonURI("/xyz_ck")) != -1);
    Assert.assertEquals(
        fileInfo, info.getClientFileInfo(info.getFileId(new TachyonURI("/xyz"))));
    Assert.assertEquals(
        ckFileInfo, info.getClientFileInfo(info.getFileId(new TachyonURI("/xyz_ck"))));
    info.stop();
  }

  /**
   * mLocalTachyonCluster is not closed in after(). Need to be closed by any test method.
   * 
   * @throws Exception
   */
  @After
  public final void after() throws Exception {
    mLocalTachyonCluster.stop();
    mExecutorService.shutdown();
    System.clearProperty("tachyon.user.quota.unit.bytes");
    System.clearProperty("fs.hdfs.impl.disable.cache");
  }

  @Before
  public final void before() throws IOException {
    System.setProperty("tachyon.user.quota.unit.bytes", "1000");
    System.setProperty("fs.hdfs.impl.disable.cache", "true");
    mLocalTachyonCluster = new LocalTachyonCluster(10000);
    mLocalTachyonCluster.start();
    mTfs = mLocalTachyonCluster.getClient();
  }

  /**
   * Test completed Editlog deletion
   * 
   * @throws Exception
   */
  @Test
  public void CompletedEditLogDeletionTest() throws Exception {
    Journal journal = mLocalTachyonCluster.getMasterInfo().getJournal();
    journal.setMaxLogSize(Constants.KB);
    for (int i = 0; i < 124; i ++) {
      mTfs.createFile(new TachyonURI("/a" + i), (i + 10) / 10 * 64);
    }
    mLocalTachyonCluster.stopTFS();
    String editLogPath = mLocalTachyonCluster.getEditLogPath();
    String completedPath =
        editLogPath.substring(0, editLogPath.lastIndexOf(TachyonURI.SEPARATOR)) + "/completed";
    Assert.assertTrue(UnderFileSystem.get(completedPath).list(completedPath).length > 1);
    MultiEditLogTestUtil();
{'fraction_non_alphanumeric': 0.11071539176215546, 'fraction_numerical': 0.007897181790027873, 'mean_word_length': 4.491921768707483, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2761455', 'n_tokens_mistral': 4167, 'n_tokens_neox': 3942, 'n_words': 894}