<commit_before>BufferList = function() {
  this.buffers = [];
};

BufferList.prototype.add = function(buffer, front) {
  this.buffers[front ? "unshift" : "push"](buffer);
  return this;
};

BufferList.prototype.addInt16 = function(val, front) {
  return this.add(Buffer([(val >>> 8),(val >>> 0)]),front);
};

BufferList.prototype.getByteLength = function(initial) {
  return this.buffers.reduce(function(previous, current){
    return previous + current.length;
  },initial || 0);
};

BufferList.prototype.addInt32 = function(val, first) {
  return this.add(Buffer([
    (val >>> 24),
    (val >>> 16),
    (val >>> 8),
    (val >>> 0)
  ]),first);
};

BufferList.prototype.addCString = function(val) {
  return this.add(Buffer(val + '\0','utf8'));
};

BufferList.prototype.addChar = function(char, first) {
  return this.add(Buffer(char,'utf8'), first);
};

BufferList.prototype.join = function(appendLength, char) {
  var length = this.getByteLength();
  if(appendLength) {
    this.addInt32(length+4, true);
    return this.join(false, char);
  }
  if(char) {
    this.addChar(char, true);
    length++;
  }
  var result = Buffer(length);
  var index = 0;
  this.buffers.forEach(function(buffer) {
    buffer.copy(result, index, 0);
    index += buffer.length;
  });
  return result;<commit_msg>changing how buffers are written<commit_after>BufferList = function() {
  this.buffers = [];
};

BufferList.prototype.add = function(buffer, front) {
  this.buffers[front ? "unshift" : "push"](buffer);
  return this;
};

BufferList.prototype.addInt16 = function(val, front) {
  return this.add(Buffer([(val >>> 8),(val >>> 0)]),front);
};

BufferList.prototype.getByteLength = function(initial) {
  return this.buffers.reduce(function(previous, current){
    return previous + current.length;
  },initial || 0);
};

BufferList.prototype.addInt32 = function(val, first) {
  return this.add(Buffer([
    (val >>> 24 & 0xFF),
    (val >>> 16 & 0xFF),
    (val >>> 8 & 0xFF),
    (val >>> 0 & 0xFF)
  ]),first);
};

BufferList.prototype.addCString = function(val) {
  return this.add(Buffer(val + '\0','utf8'));
};

BufferList.prototype.addChar = function(char, first) {
  return this.add(Buffer(char,'utf8'), first);
};

BufferList.prototype.join = function(appendLength, char) {
  var length = this.getByteLength();
  if(appendLength) {
    this.addInt32(length+4, true);
    return this.join(false, char);
  }
  if(char) {
    this.addChar(char, true);
    length++;
  }
  var result = Buffer(length);
  var index = 0;
  this.buffers.forEach(function(buffer) {
    buffer.copy(result, index, 0);
    index += buffer.length;
  });
  return result;
{'fraction_non_alphanumeric': 0.1793416572077185, 'fraction_numerical': 0.01740446462353386, 'mean_word_length': 4.133980582524272, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4110544', 'n_tokens_mistral': 927, 'n_tokens_neox': 889, 'n_words': 173}