<commit_before>package water.hdfs;

import java.io.*;
import java.net.SocketTimeoutException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.s3.S3Exception;

import water.*;
import water.api.Constants;

import com.google.common.base.Strings;
import com.google.common.io.ByteStreams;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;

/** Persistence backend for HDFS */
public abstract class PersistHdfs {
  static         final String KEY_PREFIX="";
  static         final int    KEY_PREFIX_LEN = KEY_PREFIX.length();
  static private final Configuration CONF;

  public static void initialize() { }
  static {
    Configuration conf = null;
    if( H2O.OPT_ARGS.hdfs_config != null ) {
      conf = new Configuration();
      File p = new File(H2O.OPT_ARGS.hdfs_config);
      if (!p.exists())
        Log.die("[h2o,hdfs] Unable to open hdfs configuration file "+p.getAbsolutePath());
      conf.addResource(new Path(p.getAbsolutePath()));
      System.out.println("[h2o,hdfs] resource " + p.getAbsolutePath() + " added to the hadoop configuration");
    } else {
      conf = new Configuration();
      if( !Strings.isNullOrEmpty(H2O.OPT_ARGS.hdfs) ) {
        // setup default remote Filesystem - for version 0.21 and higher
        conf.set("fs.defaultFS",H2O.OPT_ARGS.hdfs);
        // To provide compatibility with version 0.20.0 it is necessary to setup the property
        // fs.default.name which was in newer version renamed to 'fs.defaultFS'
        conf.set("fs.default.name",H2O.OPT_ARGS.hdfs);
      }
    }
    CONF = conf;
  }

  public static Configuration getConf() { return CONF; }

  public static void addFolder(Path p, JsonArray succeeded, JsonArray failed) throws IOException {
    FileSystem fs = FileSystem.get(p.toUri(), CONF);
    addFolder(fs, p, succeeded, failed);
  }

  @SuppressWarnings("deprecation")
  private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
    try {
      for( FileStatus file : fs.listStatus(p) ) {
        Path pfs = file.getPath();
        if( file.isDir() ) {
          addFolder(fs, pfs, succeeded, failed);
        } else {
          Key k = getKeyForPathString(pfs.toString());
          long size = file.getLen();
          Value val = null;
          if( pfs.getName().endsWith(".hex") ) { // Hex file?
            FSDataInputStream s = fs.open(pfs);
            int sz = (int)Math.min(1L<<20,size); // Read up to the 1st meg
            byte [] mem = MemoryManager.malloc1(sz);
            s.readFully(mem);
            // Convert to a ValueArray (hope it fits in 1Meg!)<commit_msg>Recover more gracefully from broken hdfs filename<commit_after>package water.hdfs;

import java.io.*;
import java.net.SocketTimeoutException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.s3.S3Exception;

import water.*;
import water.api.Constants;

import com.google.common.base.Strings;
import com.google.common.io.ByteStreams;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;

/** Persistence backend for HDFS */
public abstract class PersistHdfs {
  static         final String KEY_PREFIX="";
  static         final int    KEY_PREFIX_LEN = KEY_PREFIX.length();
  static private final Configuration CONF;

  public static void initialize() { }
  static {
    Configuration conf = null;
    if( H2O.OPT_ARGS.hdfs_config != null ) {
      conf = new Configuration();
      File p = new File(H2O.OPT_ARGS.hdfs_config);
      if (!p.exists())
        Log.die("[h2o,hdfs] Unable to open hdfs configuration file "+p.getAbsolutePath());
      conf.addResource(new Path(p.getAbsolutePath()));
      System.out.println("[h2o,hdfs] resource " + p.getAbsolutePath() + " added to the hadoop configuration");
    } else {
      conf = new Configuration();
      if( !Strings.isNullOrEmpty(H2O.OPT_ARGS.hdfs) ) {
        // setup default remote Filesystem - for version 0.21 and higher
        conf.set("fs.defaultFS",H2O.OPT_ARGS.hdfs);
        // To provide compatibility with version 0.20.0 it is necessary to setup the property
        // fs.default.name which was in newer version renamed to 'fs.defaultFS'
        conf.set("fs.default.name",H2O.OPT_ARGS.hdfs);
      }
    }
    CONF = conf;
  }

  public static Configuration getConf() { return CONF; }

  public static void addFolder(Path p, JsonArray succeeded, JsonArray failed) throws IOException {
    FileSystem fs = FileSystem.get(p.toUri(), CONF);
    addFolder(fs, p, succeeded, failed);
  }

  @SuppressWarnings("deprecation")
  private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
    try {
      if( fs == null ) return;
      for( FileStatus file : fs.listStatus(p) ) {
        Path pfs = file.getPath();
        if( file.isDir() ) {
          addFolder(fs, pfs, succeeded, failed);
        } else {
          Key k = getKeyForPathString(pfs.toString());
          long size = file.getLen();
          Value val = null;
          if( pfs.getName().endsWith(".hex") ) { // Hex file?
            FSDataInputStream s = fs.open(pfs);
            int sz = (int)Math.min(1L<<20,size); // Read up to the 1st meg
            byte [] mem = MemoryManager.malloc1(sz);
            s.readFully(mem);
            // Convert to a ValueArray (hope it fits in 1Meg!)
{'fraction_non_alphanumeric': 0.11972217144946079, 'fraction_numerical': 0.008042405410345458, 'mean_word_length': 2.9113652609006433, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3421409', 'n_tokens_mistral': 1687, 'n_tokens_neox': 1660, 'n_words': 443}