<commit_before>package org.apache.hadoop.hdfs.server.namenode;

import java.io.File;
import java.io.IOException;
import java.util.Random;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.ImageWriter;
import org.apache.hadoop.hdfs.server.namenode.FixedBlockResolver;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;

import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import static org.junit.Assert.*;

public class TestNNLoad {

  @Rule public TestName name = new TestName();

  final Random r = new Random();
  final File fBASE = new File(MiniDFSCluster.getBaseDirectory());
  final Path BASE = new Path(fBASE.toURI().toString());

  @Before
  public void setSeed() throws Exception {
    if (fBASE.exists() && !FileUtil.fullyDelete(fBASE)) {
      throw new IOException("Could not fully delete " + fBASE);
    }
    long seed = r.nextLong();
    r.setSeed(seed);
    System.out.println(name.getMethodName() + " seed: " + seed);
  }

  @Test(timeout = 20000)
  public void testLoadImage() throws Exception {
    final String USER = "dingo";
    final String GROUP = "yak";
    Configuration conf = new HdfsConfiguration();
    conf.set(SingleUGIResolver.USER, USER);
    conf.set(SingleUGIResolver.GROUP, GROUP);

    final Path BLOCKFILE = new Path(BASE, "blocks.csv");
    conf.set(TextFileRegionFormat.WriterOptions.FILEPATH, BLOCKFILE.toString());

    final Path NAMEPATH = new Path(BASE, "hdfs/name");
    ImageWriter.Options opts = ImageWriter.defaults();
    opts.setConf(conf);
    opts.output(NAMEPATH.toString())
        .blocks(TextFileRegionFormat.class)
        .ugi(SingleUGIResolver.class)
        .blockIds(FixedBlockResolver.class);

    final long seed = r.nextLong();
    try (ImageWriter w = new ImageWriter(opts)) {
      for (TreePath e : new RandomTreeWalk(seed)) {
        w.accept(e);
      }
    }

    conf.set(DFS_NAMENODE_NAME_DIR_KEY, NAMEPATH.toString());
    MiniDFSCluster cluster = null;
    try {
      cluster = new MiniDFSCluster.Builder(conf)
        .format(false)
        .manageNameDfsDirs(false)
        .numDataNodes(0)
        .build();
      cluster.waitActive();
      FileSystem fs = cluster.getFileSystem();
      for (TreePath e : new RandomTreeWalk(seed)) {
        FileStatus rs = e.getFileStatus();
        Path hp = new Path(rs.getPath().toUri().getPath());
        assertTrue(fs.exists(hp));
        FileStatus hs = fs.getFileStatus(hp);
        assertEquals(rs.getPath().toUri().getPath(),
                     hs.getPath().toUri().getPath());
        assertEquals(rs.getPermission(), hs.getPermission());
        // TODO: loaded later? Not reflected, yet.
        //assertEquals(rs.getReplication(), hs.getReplication());
        //assertEquals(rs.getBlockSize(), hs.getBlockSize());
        assertEquals(rs.getLen(), hs.getLen());
        assertEquals("dingo", hs.getOwner());
        assertEquals("yak", hs.getGroup());
        assertEquals(rs.getAccessTime(), hs.getAccessTime());
        assertEquals(rs.getModificationTime(), hs.getModificationTime());
      }
    } finally {
      if (cluster != null) {
        cluster.shutdown(true, true);
      }
    }
  }

}<commit_msg>Minor unit test cleanup<commit_after>package org.apache.hadoop.hdfs.server.namenode;

import java.io.File;
import java.io.IOException;
import java.util.Random;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.ImageWriter;
import org.apache.hadoop.hdfs.server.namenode.FixedBlockResolver;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;

import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import static org.junit.Assert.*;

public class TestNNLoad {

  @Rule public TestName name = new TestName();

  final Random r = new Random();
  final File fBASE = new File(MiniDFSCluster.getBaseDirectory());
  final Path BASE = new Path(fBASE.toURI().toString());

  @Before
  public void setSeed() throws Exception {
    if (fBASE.exists() && !FileUtil.fullyDelete(fBASE)) {
      throw new IOException("Could not fully delete " + fBASE);
    }
    long seed = r.nextLong();
    r.setSeed(seed);
    System.out.println(name.getMethodName() + " seed: " + seed);
  }

  @Test(timeout = 20000)
  public void testLoadImage() throws Exception {
    final String USER = "dingo";
    final String GROUP = "yak";
    Configuration conf = new HdfsConfiguration();
    conf.set(SingleUGIResolver.USER, USER);
    conf.set(SingleUGIResolver.GROUP, GROUP);

    final Path BLOCKFILE = new Path(BASE, "blocks.csv");
    conf.set(TextFileRegionFormat.WriterOptions.FILEPATH, BLOCKFILE.toString());

    final Path NAMEPATH = new Path(BASE, "hdfs/name");
    ImageWriter.Options opts = ImageWriter.defaults();
    opts.setConf(conf);
    opts.output(NAMEPATH.toString())
        .blocks(TextFileRegionFormat.class)
        .ugi(SingleUGIResolver.class)
        .blockIds(FixedBlockResolver.class);

    final long seed = r.nextLong();
    try (ImageWriter w = new ImageWriter(opts)) {
      for (TreePath e : new RandomTreeWalk(seed)) {
        w.accept(e);
      }
    }

    conf.set(DFS_NAMENODE_NAME_DIR_KEY, NAMEPATH.toString());
    MiniDFSCluster cluster = null;
    try {
      cluster = new MiniDFSCluster.Builder(conf)
        .format(false)
        .manageNameDfsDirs(false)
        .numDataNodes(0)
        .build();
      cluster.waitActive();
      FileSystem fs = cluster.getFileSystem();
      for (TreePath e : new RandomTreeWalk(seed)) {
        FileStatus rs = e.getFileStatus();
        Path hp = new Path(rs.getPath().toUri().getPath());
        assertTrue(fs.exists(hp));
        FileStatus hs = fs.getFileStatus(hp);
        assertEquals(rs.getPath().toUri().getPath(),
                     hs.getPath().toUri().getPath());
        assertEquals(rs.getPermission(), hs.getPermission());
        // TODO: loaded later? Not reflected, yet.
        //assertEquals(rs.getReplication(), hs.getReplication());
        //assertEquals(rs.getBlockSize(), hs.getBlockSize());
        assertEquals(rs.getLen(), hs.getLen());
        assertEquals(USER, hs.getOwner());
        assertEquals(GROUP, hs.getGroup());
        assertEquals(rs.getAccessTime(), hs.getAccessTime());
        assertEquals(rs.getModificationTime(), hs.getModificationTime());
      }
    } finally {
      if (cluster != null) {
        cluster.shutdown(true, true);
      }
    }
  }

}
{'fraction_non_alphanumeric': 0.13373168302745767, 'fraction_numerical': 0.0017072129748186087, 'mean_word_length': 3.39375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7665378', 'n_tokens_mistral': 2217, 'n_tokens_neox': 2159, 'n_words': 423}