<commit_before>/**
 * DNT file parser; creates a CSV file given a DNT file.
 */
import java.io.*;
import java.util.ArrayList;

class Parser {
  public static void main(String args[])
    throws IOException
  {
    if(args.length != 2) {
      System.out.println("java Parser DNTFile outputCSV");
    }

    File dntFile = new File(args[0]);
    RandomAccessFileLE ptr = new RandomAccessFileLE(dntFile, "r");

    File csvFile = new File(args[1]);
    File csvFileParent = csvFile.getParent();
    if( !csvFileParent.exists() && ! csvFileParent.mkdirs()) {
      System.out.println("Cannot create output file.");
      System.exit(1);
    }
    
    FileWriter out = new FileWriter(csvFile);

    ptr.seek(4);
    short cols = ptr.readShortLE();
    int rows = ptr.readIntLE();

    // has 1 extra column... the _Id
    short _cols = (short)(cols+1);

    String[][] dnt = new String[rows][_cols];
    String[] header = new String[_cols]; // not needed yet
    Type[] types = new Type[_cols];

    // _Id
    types[0] = Type.INT;
    header[0] = "_Id";

    out.append("_Id");
    for(int i = 0, j = 1; i < cols; i++, j++) {
      short halfWord = ptr.readShortLE();
      String val = ptr.readString(halfWord);
      
      out.append(",");
      out.append(val);
      header[j] = val;
      switch(Type.getType(ptr.readByte())) {
        case STRING: types[j] = Type.STRING; break;
        case BOOL:   types[j] = Type.BOOL;   break;
        case INT:    types[j] = Type.INT;    break;
        case FLOAT:  types[j] = Type.FLOAT;  break;
        case DOUBLE: types[j] = Type.DOUBLE; break;
        default:
          System.out.println(".dnt file is invalid!");
          ptr.close();
          System.exit(1);
          break;
      }
    }

    System.out.println(); 

    for(int i = 0; i < rows; i++) {
      out.append("\n");
      for(int j = 0; j < _cols; j++) {
        String val;
        short halfWord;
        int word;
        int dblWord;
        if(j > 0) {
          out.append(",");
        }
        switch(types[j]) {
          case STRING:  halfWord = ptr.readShortLE();
                        val = "\""+ptr.readString((int)halfWord)+"\"";
                        break;
          case BOOL:    word = ptr.readIntLE();
                        val = String.valueOf((word&1) == 1);
                        break;
          case INT:     word = ptr.readIntLE();
                        val = String.valueOf(word);
                        break;
          case FLOAT:   word = ptr.readIntLE();
                        val = String.valueOf(Float.intBitsToFloat(word));
                        break;
          case DOUBLE:  default:
                        dblWord = ptr.readIntLE();
                        val = String.valueOf(Double.longBitsToDouble(dblWord));
                        break;
        }

        dnt[i][j] = val;
        out.append(val);
      }
    }

    ptr.close();
  }
}<commit_msg>Minor optimization<commit_after>/**
 * DNT file parser; creates a CSV file given a DNT file.
 */
import java.io.*;
import java.util.ArrayList;

class Parser {
  public static void main(String args[])
    throws IOException
  {
    if(args.length != 2) {
      System.out.println("java Parser DNTFile outputCSV");
    }

    File dntFile = new File(args[0]);
    RandomAccessFileLE ptr = new RandomAccessFileLE(dntFile, "r");

    File csvFile = new File(args[1]);
    File csvFileParent = csvFile.getParent();
    if( !csvFileParent.exists() && ! csvFileParent.mkdirs()) {
      System.out.println("Cannot create output file.");
      System.exit(1);
    }

    FileWriter out = new FileWriter(csvFile);

    ptr.seek(4);
    short cols = ptr.readShortLE();
    int rows = ptr.readIntLE();

    // has 1 extra column... the _Id
    short _cols = (short)(cols+1);

    String[][] dnt = new String[rows][_cols];
    String[] header = new String[_cols]; // not needed yet
    Type[] types = new Type[_cols];

    // _Id
    types[0] = Type.INT;
    header[0] = "_Id";

    out.append("_Id");
    for(int i = 0, j = 1; i < cols; i++, j++) {
      short halfWord = ptr.readShortLE();
      String val = ptr.readString(halfWord);
      
      out.append(",");
      out.append(val);
      header[j] = val;
      switch(Type.getType(ptr.readByte())) {
        case STRING: types[j] = Type.STRING; break;
        case BOOL:   types[j] = Type.BOOL;   break;
        case INT:    types[j] = Type.INT;    break;
        case FLOAT:  types[j] = Type.FLOAT;  break;
        case DOUBLE: types[j] = Type.DOUBLE; break;
        default:
          System.out.println(".dnt file is invalid!");
          ptr.close();
          System.exit(1);
          break;
      }
    }

    System.out.println(); 

    for(int i = 0; i < rows; i++) {
      out.append("\n");
      String appendage = "";
      for(int j = 0; j < _cols; j++) {
        String val;
        short halfWord;
        int word;
        int dblWord;
        out.append(appendage);
        switch(types[j]) {
          case STRING:  halfWord = ptr.readShortLE();
                        val = "\""+ptr.readString((int)halfWord)+"\"";
                        break;
          case BOOL:    word = ptr.readIntLE();
                        val = String.valueOf((word&1) == 1);
                        break;
          case INT:     word = ptr.readIntLE();
                        val = String.valueOf(word);
                        break;
          case FLOAT:   word = ptr.readIntLE();
                        val = String.valueOf(Float.intBitsToFloat(word));
                        break;
          case DOUBLE:  default:
                        dblWord = ptr.readIntLE();
                        val = String.valueOf(Double.longBitsToDouble(dblWord));
                        break;
        }

        dnt[i][j] = val;
        out.append(val);
        appendage = ",";
      }
    }

    ptr.close();
  }
}
{'fraction_non_alphanumeric': 0.14275991151948272, 'fraction_numerical': 0.005615109749872384, 'mean_word_length': 1.7012867647058822, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4774677', 'n_tokens_mistral': 1869, 'n_tokens_neox': 1794, 'n_words': 434}