<commit_before>/*
 *    Copyright 2016 Rethink Robotics
 *
 *    Copyright 2016 <NAME>
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

'use strict'

//-----------------------------------------------------------------------------
// Base Type Deserializer Functions
//-----------------------------------------------------------------------------

function DefaultArrayDeserializer(deserializeFunc, buffer, bufferOffset, array) {
  for (let i = 0; i < array.length; ++i) {
    array[i] = deserializeFunc(buffer, bufferOffset, null);
  }
  return array;
}

function StringDeserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(StringDeserializer, buffer, bufferOffset, array);
  }

  let len = buffer.readUInt32LE(bufferOffset[0]);
  //console.log('len ' + len);
  bufferOffset[0] += 4;
  let str = buffer.slice(bufferOffset[0], bufferOffset[0] + len).toString();
  bufferOffset[0] += len;
  return str;
}

function UInt8Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(UInt8Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readUInt8(bufferOffset[0], true);
  bufferOffset[0] += 1;
  return val;
}

function UInt16Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(UInt16Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readUInt16LE(bufferOffset[0], true);
  bufferOffset[0] += 2;
  return val;
}

function UInt32Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(UInt32Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readUInt32LE(bufferOffset[0]);
  bufferOffset[0] += 4;
  return val;
}

function UInt64Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(UInt64Deserializer, buffer, bufferOffset, array);
  }

  // FIXME: best way to do this??
  let val = buffer.slice(bufferOffset[0], 8)
  bufferOffset[0] += 8;
  return val;
}

function Int8Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Int8Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readInt8(bufferOffset[0], true);
  bufferOffset[0] += 1;
  return val;
}

function Int16Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Int16Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readInt16LE(bufferOffset[0], true);
  bufferOffset[0] += 2;
  return val;
}

function Int32Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Int32Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readInt32LE(bufferOffset[0], true);
  bufferOffset[0] += 4;
  return val;
}

function Int64Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Int64Deserializer, buffer, bufferOffset, array);
  }

  // FIXME: best way to do this??
  let val = buffer.slice(bufferOffset[0], 8);
  bufferOffset[0] += 8;
  return val;
}

function Float32Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Float32Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readFloatLE(bufferOffset[0], true);
  bufferOffset[0] += 4;
  return val;
}

function Float64Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Float64Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readDoubleLE(bufferOffset[0], true);
  bufferOffset[0] += 8;
  return val;
}

function TimeDeserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(TimeDeserializer, buffer, bufferOffset, array);
  }

  let secs = Int32Deserializer(buffer, bufferOffset);
  let nsecs = Int32Deserializer(buffer, bufferOffset);<commit_msg>Fix 64-bit integer slice calls<commit_after>/*
 *    Copyright 2016 Rethink Robotics
 *
 *    Copyright 2016 <NAME>
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

'use strict'

//-----------------------------------------------------------------------------
// Base Type Deserializer Functions
//-----------------------------------------------------------------------------

function DefaultArrayDeserializer(deserializeFunc, buffer, bufferOffset, array) {
  for (let i = 0; i < array.length; ++i) {
    array[i] = deserializeFunc(buffer, bufferOffset, null);
  }
  return array;
}

function StringDeserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(StringDeserializer, buffer, bufferOffset, array);
  }

  let len = buffer.readUInt32LE(bufferOffset[0]);
  //console.log('len ' + len);
  bufferOffset[0] += 4;
  let str = buffer.slice(bufferOffset[0], bufferOffset[0] + len).toString();
  bufferOffset[0] += len;
  return str;
}

function UInt8Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(UInt8Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readUInt8(bufferOffset[0], true);
  bufferOffset[0] += 1;
  return val;
}

function UInt16Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(UInt16Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readUInt16LE(bufferOffset[0], true);
  bufferOffset[0] += 2;
  return val;
}

function UInt32Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(UInt32Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readUInt32LE(bufferOffset[0]);
  bufferOffset[0] += 4;
  return val;
}

function UInt64Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(UInt64Deserializer, buffer, bufferOffset, array);
  }

  // FIXME: best way to do this??
  let val = buffer.slice(bufferOffset[0], bufferOffset[0] + 8)
  bufferOffset[0] += 8;
  return val;
}

function Int8Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Int8Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readInt8(bufferOffset[0], true);
  bufferOffset[0] += 1;
  return val;
}

function Int16Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Int16Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readInt16LE(bufferOffset[0], true);
  bufferOffset[0] += 2;
  return val;
}

function Int32Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Int32Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readInt32LE(bufferOffset[0], true);
  bufferOffset[0] += 4;
  return val;
}

function Int64Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Int64Deserializer, buffer, bufferOffset, array);
  }

  // FIXME: best way to do this??
  let val = buffer.slice(bufferOffset[0], bufferOffset[0] + 8);
  bufferOffset[0] += 8;
  return val;
}

function Float32Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Float32Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readFloatLE(bufferOffset[0], true);
  bufferOffset[0] += 4;
  return val;
}

function Float64Deserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(Float64Deserializer, buffer, bufferOffset, array);
  }

  let val = buffer.readDoubleLE(bufferOffset[0], true);
  bufferOffset[0] += 8;
  return val;
}

function TimeDeserializer(buffer, bufferOffset, array) {
  if (array) {
    return DefaultArrayDeserializer(TimeDeserializer, buffer, bufferOffset, array);
  }

  let secs = Int32Deserializer(buffer, bufferOffset);
  let nsecs = Int32Deserializer(buffer, bufferOffset);
{'fraction_non_alphanumeric': 0.1421153629077774, 'fraction_numerical': 0.023704707077548254, 'mean_word_length': 3.8152173913043477, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5594885', 'n_tokens_mistral': 2799, 'n_tokens_neox': 2688, 'n_words': 719}