<commit_before>package embeddings;

import java.util.HashMap;
import static embeddings.EmbeddingsParser.NBYTES;
import static java.nio.charset.StandardCharsets.UTF_8;


/**
 * Sample class for using word embeddings.
 * Provides an enum that statically loads embeddings
 */
public class WordEmbeddings {
  private float _scale;
  private int _shift;
  private short _vec_sz;

  public enum EMBEDDINGS {
    GLOVE(EmbeddingsParser.parse("./data/glove.bin")),
    GOOGL(EmbeddingsParser.parse("./data/googl.bin"));

    private final WordEmbeddings _em;
    private EMBEDDINGS(EmbeddingsParser ep) {
      _em=new WordEmbeddings(ep._maps);
      _em._scale = 1.f/(float)Math.pow(10,ep._scale);
      _em._shift = ep._shift;
      _em._vec_sz = ep._vec_sz;
    }

    /**
     * Fill res with the word embeddings for word w.
     * The expectation is that res is zero'd out before use.
     * This will force the word into UTF_8 bytes.
     * This will allocate a new BufferedBytes instance to wrap the bytes of w.
     *
     * @param w lookup w in the word embeddings
     * @param res fill this float array with word embeddings
     */
    public void get(String w, float[] res) {
      _em.get(new BufferedBytes(w.getBytes(UTF_8)),res);
    }

    /**
     * Fill res with the word embeddings for word w.
     * The expectation is that res is zero'd out before use.
     * This allocates a new BufferedBytes instance to wrap w.
     *
     * @param w lookup w in the word embeddings
     * @param res fill this float array with word embeddings
     */
    public void get(byte[] w, float[] res) {
      _em.get(new BufferedBytes(w),res);
    }


    /**
     * Fill res with the word embeddings for the BufferedBytes instance.
     * This is the best performing option.
     * The expectation is that res is zero'd out before use.
     *
     * @param bb BufferedBytes holding the bytes of the word to lookup
     * @param res fill this float array with the word embeddings
     */
    public void get(BufferedBytes bb, float[] res) {
      _em.get(bb, res);
    }
  }

  private transient HashMap<BufferedBytes,BufferedBytes> _map;
  // takes the result of a parse_bin call and flattens all maps into a single map
  private WordEmbeddings(HashMap<BufferedBytes,BufferedBytes>[] maps) {
    _map=new HashMap<>();
    for (HashMap<BufferedBytes, BufferedBytes> map : maps) _map.putAll(map);
  }

  private void get(BufferedBytes s, float[] res) {
    BufferedBytes bb= _map.get(s);
    if( bb==null )
      return;
    int off=bb._off + bb._len; // _off is the start of the string, _len is the length of the string
    byte[] buf = bb._buf;
    int idx=0;
    int i=off;
    for(;i<off+NBYTES*_vec_sz;i+=NBYTES) {
      // decode the embedding value by combining 3 bytes, then shift & scale
      int r = (buf[i  ] & 0xFF)      |
              (buf[i+1] & 0xFF) << 8 |
              (buf[i+2] & 0xFF) << 16;
      res[idx++] = ((r + _shift)*_scale);
    }
  }
}<commit_msg>don't rely on caller to always 0 out reslt array<commit_after>package embeddings;

import java.util.Arrays;
import java.util.HashMap;

import static embeddings.EmbeddingsParser.NBYTES;
import static java.nio.charset.StandardCharsets.UTF_8;


/**
 * Sample class for using word embeddings.
 * Provides an enum that statically loads embeddings
 */
public class WordEmbeddings {
  private float _scale;
  private int _shift;
  private short _vec_sz;

  public enum EMBEDDINGS {
    GLOVE(EmbeddingsParser.parse("./data/glove.bin")),
    GOOGL(EmbeddingsParser.parse("./data/googl.bin"));

    private final WordEmbeddings _em;
    private EMBEDDINGS(EmbeddingsParser ep) {
      _em=new WordEmbeddings(ep._maps);
      _em._scale = 1.f/(float)Math.pow(10,ep._scale);
      _em._shift = ep._shift;
      _em._vec_sz = ep._vec_sz;
    }

    /**
     * Fill res with the word embeddings for word w.
     * The expectation is that res is zero'd out before use.
     * This will force the word into UTF_8 bytes.
     * This will allocate a new BufferedBytes instance to wrap the bytes of w.
     *
     * @param w lookup w in the word embeddings
     * @param res fill this float array with word embeddings
     */
    public void get(String w, float[] res) {
      _em.get(new BufferedBytes(w.getBytes(UTF_8)),res);
    }

    /**
     * Fill res with the word embeddings for word w.
     * The expectation is that res is zero'd out before use.
     * This allocates a new BufferedBytes instance to wrap w.
     *
     * @param w lookup w in the word embeddings
     * @param res fill this float array with word embeddings
     */
    public void get(byte[] w, float[] res) {
      _em.get(new BufferedBytes(w),res);
    }


    /**
     * Fill res with the word embeddings for the BufferedBytes instance.
     * This is the best performing option.
     * The expectation is that res is zero'd out before use.
     *
     * @param bb BufferedBytes holding the bytes of the word to lookup
     * @param res fill this float array with the word embeddings
     */
    public void get(BufferedBytes bb, float[] res) {
      _em.get(bb, res);
    }
  }

  private transient HashMap<BufferedBytes,BufferedBytes> _map;
  // takes the result of a parse_bin call and flattens all maps into a single map
  private WordEmbeddings(HashMap<BufferedBytes,BufferedBytes>[] maps) {
    _map=new HashMap<>();
    for (HashMap<BufferedBytes, BufferedBytes> map : maps) _map.putAll(map);
  }

  private void get(BufferedBytes s, float[] res) {
    Arrays.fill(res,0);
    BufferedBytes bb= _map.get(s);
    if( bb==null )
      return;
    int off=bb._off + bb._len; // _off is the start of the string, _len is the length of the string
    byte[] buf = bb._buf;
    int idx=0;
    int i=off;
    for(;i<off+NBYTES*_vec_sz;i+=NBYTES) {
      // decode the embedding value by combining 3 bytes, then shift & scale
      int r = (buf[i  ] & 0xFF)      |
              (buf[i+1] & 0xFF) << 8 |
              (buf[i+2] & 0xFF) << 16;
      res[idx++] = ((r + _shift)*_scale);
    }
  }
}
{'fraction_non_alphanumeric': 0.10068004644219605, 'fraction_numerical': 0.005639409520650191, 'mean_word_length': 2.7616968184653774, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1571', 'n_tokens_mistral': 1968, 'n_tokens_neox': 1865, 'n_words': 650}