<commit_before>package edu.berkeley.cs.succinct.examples;

import edu.berkeley.cs.succinct.StorageMode;
import edu.berkeley.cs.succinct.buffers.SuccinctFileBuffer;
import edu.berkeley.cs.succinct.buffers.SuccinctIndexedFileBuffer;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;

public class Construct {
  public static void main(String[] args) throws IOException {
    if (args.length < 2 || args.length > 3) {
      System.err.println("Parameters: [input-path] [output-path] <[type]>");
      System.exit(-1);
    }

    SuccinctFileBuffer succinctFileBuffer = null;

    String type = "file";
    if (args.length == 3) {
      type = args[2];
    }

    if (type == "file") {
      if (args[0].endsWith(".succinct")) {
        succinctFileBuffer = new SuccinctFileBuffer(args[0], StorageMode.MEMORY_ONLY);
      } else {
        File file = new File(args[0]);
        if (file.length() > 1L << 31) {
          System.err.println("Cant handle files > 2GB");
          System.exit(-1);
        }
        byte[] fileData = new byte[(int) file.length()];
        System.out.println("File size: " + fileData.length + " bytes");
        DataInputStream dis = new DataInputStream(new FileInputStream(file));
        dis.readFully(fileData, 0, (int) file.length());

        succinctFileBuffer = new SuccinctFileBuffer(fileData);
      }
    } else if (type == "indexed-file") {
      if (args[0].endsWith(".succinct")) {
        succinctFileBuffer = new SuccinctIndexedFileBuffer(args[0], StorageMode.MEMORY_ONLY);
      } else {
        File file = new File(args[0]);
        if (file.length() > 1L << 31) {
          System.err.println("Cant handle files > 2GB");
          System.exit(-1);
        }
        byte[] fileData = new byte[(int) file.length()];
        System.out.println("File size: " + fileData.length + " bytes");
        DataInputStream dis = new DataInputStream(new FileInputStream(file));
        dis.readFully(fileData, 0, (int) file.length());

        ArrayList<Integer> positions = new ArrayList<Integer>();
        positions.add(0);
        for (int i = 0; i < fileData.length; i++) {
          if (fileData[i] == '\n') {
            positions.add(i + 1);
          }
        }
        int[] offsets = new int[positions.size()];

        succinctFileBuffer = new SuccinctIndexedFileBuffer(fileData, offsets);
      }
    }

    assert succinctFileBuffer != null;

    succinctFileBuffer.writeToFile(args[1]);
  }
}
<commit_msg>Fixed Construct example to populate offsets array<commit_after>package edu.berkeley.cs.succinct.examples;

import edu.berkeley.cs.succinct.StorageMode;
import edu.berkeley.cs.succinct.buffers.SuccinctFileBuffer;
import edu.berkeley.cs.succinct.buffers.SuccinctIndexedFileBuffer;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;

public class Construct {
  public static void main(String[] args) throws IOException {
    if (args.length < 2 || args.length > 3) {
      System.err.println("Parameters: [input-path] [output-path] <[type]>");
      System.exit(-1);
    }

    SuccinctFileBuffer succinctFileBuffer = null;

    String type = "file";
    if (args.length == 3) {
      type = args[2];
    }

    if (type == "file") {
      if (args[0].endsWith(".succinct")) {
        succinctFileBuffer = new SuccinctFileBuffer(args[0], StorageMode.MEMORY_ONLY);
      } else {
        File file = new File(args[0]);
        if (file.length() > 1L << 31) {
          System.err.println("Cant handle files > 2GB");
          System.exit(-1);
        }
        byte[] fileData = new byte[(int) file.length()];
        System.out.println("File size: " + fileData.length + " bytes");
        DataInputStream dis = new DataInputStream(new FileInputStream(file));
        dis.readFully(fileData, 0, (int) file.length());

        succinctFileBuffer = new SuccinctFileBuffer(fileData);
      }
    } else if (type == "indexed-file") {
      if (args[0].endsWith(".succinct")) {
        succinctFileBuffer = new SuccinctIndexedFileBuffer(args[0], StorageMode.MEMORY_ONLY);
      } else {
        File file = new File(args[0]);
        if (file.length() > 1L << 31) {
          System.err.println("Cant handle files > 2GB");
          System.exit(-1);
        }
        byte[] fileData = new byte[(int) file.length()];
        System.out.println("File size: " + fileData.length + " bytes");
        DataInputStream dis = new DataInputStream(new FileInputStream(file));
        dis.readFully(fileData, 0, (int) file.length());

        ArrayList<Integer> positions = new ArrayList<Integer>();
        positions.add(0);
        for (int i = 0; i < fileData.length; i++) {
          if (fileData[i] == '\n') {
            positions.add(i + 1);
          }
        }
        int[] offsets = new int[positions.size()];
        for (int i = 0; i < offsets.length; i++) {
          offsets[i] = positions.get(i);
        }
        succinctFileBuffer = new SuccinctIndexedFileBuffer(fileData, offsets);
      }
    }

    assert succinctFileBuffer != null;

    succinctFileBuffer.writeToFile(args[1]);
  }
}

{'fraction_non_alphanumeric': 0.1431578947368421, 'fraction_numerical': 0.010526315789473684, 'mean_word_length': 2.7489239598278337, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3956681', 'n_tokens_mistral': 1677, 'n_tokens_neox': 1605, 'n_words': 343}