<commit_before>
onmessage = function(e) {
  var command = e.command;
  var packed = command.compressedBytes;
  var buf_length = packed.length;
  var buf_pos = 0;
  var buf = new Uint8Array(buf);
  function checkOutputSpace(n) {
    if (buf_pos + n > buf_length) {
      buf_length *= 2;
      var new_buf = new Uint8Array(buf_length);
      new_buf.set(buf);
      buf = new_buf;
    }
  }
  var pos = 0;
  while (pos < packed.length) {
    var b = packed[pos++];
    if (b & 0x80) {
      if (b === 0x80) continue;
      var count = 1 - (b << 24 >> 24);
      var rep = packed[pos++];
      checkOutputSpace(count);<commit_msg>Simplify signed byte calc to a  unsigned one<commit_after>
onmessage = function(e) {
  var command = e.command;
  var packed = command.compressedBytes;
  var buf_length = packed.length;
  var buf_pos = 0;
  var buf = new Uint8Array(buf);
  function checkOutputSpace(n) {
    if (buf_pos + n > buf_length) {
      buf_length *= 2;
      var new_buf = new Uint8Array(buf_length);
      new_buf.set(buf);
      buf = new_buf;
    }
  }
  var pos = 0;
  while (pos < packed.length) {
    var b = packed[pos++];
    if (b & 0x80) {
      if (b === 0x80) continue;
      var count = 257 - b;
      var rep = packed[pos++];
      checkOutputSpace(count);
{'fraction_non_alphanumeric': 0.13019607843137254, 'fraction_numerical': 0.023529411764705882, 'mean_word_length': 2.5943661971830987, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2048008', 'n_tokens_mistral': 482, 'n_tokens_neox': 457, 'n_words': 124}