<commit_before>// copyright 2009 ActiveVideo; license: MIT; see license.txt
package mmeta;

import java.util.*;

class MemoStat {
  public String name;
  public long stores;
  public long hits;
  public long misses;
  public boolean recursion;
}

class State {
    State prev = null;
    int pos; Object[] list; SparseArrayList<HashMap<String, Memoize>> positions;
    public State(State prev, int p, Object[] l, SparseArrayList<HashMap<String, Memoize>> m) {
        this.prev = prev;
        pos = p; list = l; positions = m;
    }
}

class Memoize {
    public Object val; public int pos; int seed = -1;
    public Memoize(Object val, int pos) { this.val = val; this.pos = pos; }
    public String toString() { return "at: "+ pos +" val: "+ val; }
}

/// Root parser object, all parsers will in the end extend BaseParser
public class BaseParser {
    public BaseParser() {
        lines.put(0, 0);
    }

    private static class ReverseComparator implements Comparator<Comparable>  {
        public int compare(Comparable a, Comparable b) {
            return -a.compareTo(b);
        }
    }

    public static String print_r(Object o) {
        StringBuffer sb = new StringBuffer();
        print_r(o, sb);
        return sb.toString();
    }
    public static void print_r(Object o, StringBuffer sb) {
        if (o instanceof List) {
            sb.append("[");
            for (int i = 0; i < ((List)o).size(); i++) {
                if (i > 0) sb.append(", ");
                print_r(((List)o).get(i), sb);
            }
            sb.append("]");
        } else if (o instanceof Object[]) {
            sb.append("[");
            for (int i = 0; i < ((Object[])o).length; i++) {
                if (i > 0) sb.append(", ");
                print_r(((Object[])o)[i], sb);
            }
            sb.append("]");
        } else {
            sb.append(o);
        }
    }
    public void _enter(String label) {
      if (debug_parse_tree) {
        String parent = parseTree.peekLast();
        if ("ws".equals(label) ||
            "<skip>".equals(label) ||
            "skip".equals(parent)) {
          parseTree.addLast("skip");<commit_msg>ws, javadoc ish comments in BaseParser<commit_after>// copyright 2009 ActiveVideo; license: MIT; see license.txt
package mmeta;

import java.util.*;

class MemoStat {
  public String name;
  public long stores;
  public long hits;
  public long misses;
  public boolean recursion;
}

class State {
    State prev = null;
    int pos; Object[] list; SparseArrayList<HashMap<String, Memoize>> positions;
    public State(State prev, int p, Object[] l, SparseArrayList<HashMap<String, Memoize>> m) {
        this.prev = prev;
        pos = p; list = l; positions = m;
    }
}

class Memoize {
    public Object val; public int pos; int seed = -1;
    public Memoize(Object val, int pos) { this.val = val; this.pos = pos; }
    public String toString() { return "at: "+ pos +" val: "+ val; }
}

/**
 * Root parser object, all parsers will in the end extend BaseParser.
 */
public class BaseParser {
    public BaseParser() {
        lines.put(0, 0);
    }

    private static class ReverseComparator implements Comparator<Comparable>  {
        public int compare(Comparable a, Comparable b) {
            return -a.compareTo(b);
        }
    }

    /**
     * Prints a representation of the passed object.
     */
    public static String print_r(Object o) {
        StringBuffer sb = new StringBuffer();
        print_r(o, sb);
        return sb.toString();
    }

    public static void print_r(Object o, StringBuffer sb) {
        if (o instanceof List) {
            sb.append("[");
            for (int i = 0; i < ((List)o).size(); i++) {
                if (i > 0) sb.append(", ");
                print_r(((List)o).get(i), sb);
            }
            sb.append("]");
        } else if (o instanceof Object[]) {
            sb.append("[");
            for (int i = 0; i < ((Object[])o).length; i++) {
                if (i > 0) sb.append(", ");
                print_r(((Object[])o)[i], sb);
            }
            sb.append("]");
        } else {
            sb.append(o);
        }
    }

    public void _enter(String label) {
      if (debug_parse_tree) {
        String parent = parseTree.peekLast();
        if ("ws".equals(label) ||
            "<skip>".equals(label) ||
            "skip".equals(parent)) {
          parseTree.addLast("skip");
{'fraction_non_alphanumeric': 0.13897691263014939, 'fraction_numerical': 0.004979628791308284, 'mean_word_length': 2.0623700623700625, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7422191', 'n_tokens_mistral': 1349, 'n_tokens_neox': 1326, 'n_words': 399}