<commit_before>package edu.jhu.thrax.extraction;

import java.util.HashMap;

import edu.jhu.thrax.util.Vocabulary;

public enum LabelCache {
  SLASH("/"), BACKSLASH("\\"), PLUS("+");
  
  private HashMap<Long, Integer> cache = new HashMap<Long, Integer>();
  private String glue;
  
  private LabelCache(String g) {
    glue = g;
  }
  
  public int get(int left, int right) {
    System.err.println("LEFT: " + Integer.toBinaryString(left));
    System.err.println("RGHT: " + Integer.toBinaryString(right));
    
    long key = ((long) left << 32) | ((long) right & 0x00000000FFFFFFFFL);
    
    System.err.println("SHFT: " + Long.toBinaryString(((long) left << 32)));
    System.err.println("JOIN: " + Long.toBinaryString(key));
    
    Integer val = cache.get(key);
    if (val == null) {
      val = join(left, right, glue);
      cache.put(key, val);
    }
    return val;
  }
  
  private static final int join(int a, int b, String glue) {
    String word_a = Vocabulary.word(a);
    String word_b = Vocabulary.word(b);
    return Vocabulary.id(word_a.substring(0, word_a.length() - 1) + glue<commit_msg>Protip: take your debugging println's out before you run big jobs.<commit_after>package edu.jhu.thrax.extraction;

import java.util.HashMap;

import edu.jhu.thrax.util.Vocabulary;

public enum LabelCache {
  SLASH("/"), BACKSLASH("\\"), PLUS("+");
  
  private HashMap<Long, Integer> cache = new HashMap<Long, Integer>();
  private String glue;
  
  private LabelCache(String g) {
    glue = g;
  }
  
  public int get(int left, int right) {
    long key = ((long) left << 32) | ((long) right & 0x00000000FFFFFFFFL);
    Integer val = cache.get(key);
    if (val == null) {
      val = join(left, right, glue);
      cache.put(key, val);
    }
    return val;
  }
  
  private static final int join(int a, int b, String glue) {
    String word_a = Vocabulary.word(a);
    String word_b = Vocabulary.word(b);
    return Vocabulary.id(word_a.substring(0, word_a.length() - 1) + glue
{'fraction_non_alphanumeric': 0.14917127071823205, 'fraction_numerical': 0.014063284781516826, 'mean_word_length': 3.293103448275862, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4323023', 'n_tokens_mistral': 730, 'n_tokens_neox': 689, 'n_words': 171}