<commit_before>/// <reference path='../../../third_party/typings/browser.d.ts' />

// Byte-wise equality check of array buffers by comparison of each byte's
// value.
export function byteEquality(b1 :ArrayBuffer, b2 :ArrayBuffer)
    :boolean {
  var a1 = new Uint8Array(b1);
  var a2 = new Uint8Array(b2);
  if(a1.byteLength !== a2.byteLength) return false;
  for(var i:number = 0; i < a1.byteLength; ++i) {
    if(a1[i] !== a2[i]) return false;
  }
  return true;
}

// Concat |ArrayBuffer|s into a single ArrayBuffer. If size is given, then
// the destination array buffer is of the given size. If size is not given or
// zero, the  size of all buffers is summed to make the new array buffer.
export function concat(buffers:ArrayBuffer[], size?:number)
    :ArrayBuffer {
  if(!size) {
    size = 0;
    buffers.forEach(a => { size += a.byteLength });
  }
  var accumulatorBuffer = new Uint8Array(size);<commit_msg>farm out arraybuffers.byteEquality to Buffer<commit_after>/// <reference path='../../../third_party/typings/browser.d.ts' />

// Returns true if b1 and b2 have exactly the same bytes.
export function byteEquality(b1: ArrayBuffer, b2: ArrayBuffer): boolean {
  // The Buffer instances share memory with their source ArrayBuffers.
  return new Buffer(b1).equals(new Buffer(b2));
}

// Concat |ArrayBuffer|s into a single ArrayBuffer. If size is given, then
// the destination array buffer is of the given size. If size is not given or
// zero, the  size of all buffers is summed to make the new array buffer.
export function concat(buffers:ArrayBuffer[], size?:number)
    :ArrayBuffer {
  if(!size) {
    size = 0;
    buffers.forEach(a => { size += a.byteLength });
  }
  var accumulatorBuffer = new Uint8Array(size);
{'fraction_non_alphanumeric': 0.1250720461095101, 'fraction_numerical': 0.0138328530259366, 'mean_word_length': 3.641711229946524, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5909906', 'n_tokens_mistral': 549, 'n_tokens_neox': 546, 'n_words': 193}