<commit_before>package edu.jhu.thrax.tools;

import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.net.URI;
import java.util.logging.Logger;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;

import edu.jhu.jerboa.util.FileManager;
import edu.jhu.thrax.hadoop.distributional.SignatureWritable;

public class SequenceToSignatures {

  private static final Logger logger = Logger.getLogger(SequenceToSignatures.class.getName());

  public static void main(String[] args) throws Exception {

    boolean local = true;
    String input_file = null;
    int chunk_size = 500000;
    String output_prefix = null;

    for (int i = 0; i < args.length; i++) {
      if ("-i".equals(args[i]) && (i < args.length - 1)) {
        input_file = args[++i];
      } else if ("-o".equals(args[i]) && (i < args.length - 1)) {
        output_prefix = args[++i];
      } else if ("-c".equals(args[i]) && (i < args.length - 1)) {
        chunk_size = Integer.parseInt(args[++i]);
      }
    }
    if (input_file == null) {
      logger.severe("No input file specified.");
      System.exit(0);
    }
    if (output_prefix == null) {
      logger.severe("No output prefix specified.");
      System.exit(0);
    }

    logger.info("Looking for " + input_file + " on " + (local ? "local filesystem" : "HDFS") + ".");

    Configuration config = new Configuration();
    SignatureWritable signature = new SignatureWritable();
    
    
    SequenceFile.Reader reader;
    if (local) {
      Path path = new Path(input_file);
      reader = new SequenceFile.Reader(FileSystem.getLocal(config), path, config);
    } else {
      // TODO: Only works for completely specified URLs (i.e. hdfs://name-node/...), currently
      // disabled until I figure out how to get simple paths to work in HDFS.
      FileSystem file_system = FileSystem.get(URI.create(input_file), config);
      Path path = new Path(input_file);
      reader = new SequenceFile.Reader(file_system, path, config);
    }

    int chunk_id = 0;
    int key_count = 0;

    FileOutputStream bytes_out = null;
    BufferedWriter strengths_writer = null;
    BufferedWriter keys_writer = null;

    while (reader.next(signature)) {
      if (key_count % chunk_size == 0) {
        if (key_count != 0) {
          keys_writer.close();
          bytes_out.close();
          strengths_writer.close();
        }
        String chunk_tag = String.format("-%05d", chunk_id);
        bytes_out = new FileOutputStream(output_prefix + chunk_tag + ".bytes");
        strengths_writer = FileManager.getWriter(output_prefix + chunk_tag + ".strengths.gz");
        keys_writer = FileManager.getWriter(output_prefix + chunk_tag + ".keys.gz");
        chunk_id++;
      }
      keys_writer.write(signature.key.toString());
      keys_writer.newLine();
      bytes_out.write(signature.bytes.getBytes());
      strengths_writer.write("" + signature.strength.get());
      strengths_writer.newLine();
      key_count++;
    }
    reader.close();
    keys_writer.close();
    bytes_out.close();
    strengths_writer.close();
  }<commit_msg>Added writing of config.<commit_after>package edu.jhu.thrax.tools;

import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.net.URI;
import java.util.logging.Logger;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;

import edu.jhu.jerboa.util.FileManager;
import edu.jhu.thrax.hadoop.distributional.SignatureWritable;

public class SequenceToSignatures {

  private static final Logger logger = Logger.getLogger(SequenceToSignatures.class.getName());

  private static void writeConfig(String config_file, int num_bits) throws IOException {
    ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(config_file));
    out.writeInt(num_bits);
    // Does not contain support.
    out.writeBoolean(false);
    out.flush();
    out.close();
  }

  public static void main(String[] args) throws Exception {
    boolean local = true;
    String input_file = null;
    int chunk_size = 500000;
    String output_prefix = null;

    for (int i = 0; i < args.length; i++) {
      if ("-i".equals(args[i]) && (i < args.length - 1)) {
        input_file = args[++i];
      } else if ("-o".equals(args[i]) && (i < args.length - 1)) {
        output_prefix = args[++i];
      } else if ("-c".equals(args[i]) && (i < args.length - 1)) {
        chunk_size = Integer.parseInt(args[++i]);
      }
    }
    if (input_file == null) {
      logger.severe("No input file specified.");
      System.exit(0);
    }
    if (output_prefix == null) {
      logger.severe("No output prefix specified.");
      System.exit(0);
    }

    logger.info("Looking for " + input_file + " on " + (local ? "local filesystem" : "HDFS") + ".");

    Configuration config = new Configuration();
    SignatureWritable signature = new SignatureWritable();


    SequenceFile.Reader reader;
    if (local) {
      Path path = new Path(input_file);
      reader = new SequenceFile.Reader(FileSystem.getLocal(config), path, config);
    } else {
      // TODO: Only works for completely specified URLs (i.e. hdfs://name-node/...), currently
      // disabled until I figure out how to get simple paths to work in HDFS.
      FileSystem file_system = FileSystem.get(URI.create(input_file), config);
      Path path = new Path(input_file);
      reader = new SequenceFile.Reader(file_system, path, config);
    }

    int chunk_id = 0;
    int key_count = 0;

    FileOutputStream bytes_out = null;
    BufferedWriter strengths_writer = null;
    BufferedWriter keys_writer = null;

    while (reader.next(signature)) {
      if (key_count % chunk_size == 0) {
        if (key_count != 0) {
          keys_writer.close();
          bytes_out.close();
          strengths_writer.close();
        }
        String chunk_tag = String.format("-%05d", chunk_id);
        writeConfig(output_prefix + chunk_tag + ".config", signature.bytes.getBytes().length * 8);
        bytes_out = new FileOutputStream(output_prefix + chunk_tag + ".bytes");
        strengths_writer = FileManager.getWriter(output_prefix + chunk_tag + ".strengths.gz");
        keys_writer = FileManager.getWriter(output_prefix + chunk_tag + ".keys.gz");
        chunk_id++;
      }
      keys_writer.write(signature.key.toString());
      keys_writer.newLine();
      bytes_out.write(signature.bytes.getBytes());
      strengths_writer.write("" + signature.strength.get());
      strengths_writer.newLine();
      key_count++;
    }
    reader.close();
    keys_writer.close();
    bytes_out.close();
    strengths_writer.close();
  }
{'fraction_non_alphanumeric': 0.13294460641399417, 'fraction_numerical': 0.005393586005830904, 'mean_word_length': 3.06457345971564, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5622294', 'n_tokens_mistral': 2135, 'n_tokens_neox': 2072, 'n_words': 472}