<commit_before>var sys = require('sys')
  , lib = require('./lib.js')
  ;

function is_int(val) {
  return (!isNaN(val)) && (parseFloat(val) === parseInt(val));
}

// Return the "meat" of a regex, or null if it is not a regex.
function regex_of(val) {
  return lib.typeOf(val) === 'regexp' ? val.toString().slice(1, -1) : null;
}

// Use object creation because I don't like object literal syntax to place functions in a namespace.
var Encoder = function() {
  var self = this;

  this.encode = function(term) {
    var encoder = this[lib.typeOf(term)];
    if(!encoder)
      throw new Error("Do not know how to encode " + lib.typeOf(term) + ': ' + sys.inspect(term));
    return encoder.apply(self, [term]);
  }

  this.number = function(x) {
    return is_int(x) ? this.int(x) : this.float(x);
  }

  this.int = function(x) {
    if(x >= 0 && x < 256)
      return [lib.tags.SMALL_INTEGER, x];
    else
      throw new Error('Unknown integer: ' + x);
  }

  this.raw_array = function(x) {
    // Simple array encoding, without worrying about tagging.
    var result = [];
    if(x.length) {
      result.push( lib.tags.LIST
                 , lib.uint32(x.length)
                 , x.map(function(e) { return self.encode(e) }));
    }
    result.push(lib.tags.NIL);
    return result;
  }

  this.array = function(x) {
    if(x.length !== 2)
      // All arays not of length 2 are innocent.
      return self.raw_array(x);

    // 2-Arrays are used to encode special-cases where Javascript has no syntax to match Erlang.
    var tag = x[0], val = x[1];
    var re = regex_of(tag);

    if(!re)
      // Nope, just an innocent 2-array.
      return self.raw_array(x);

    if((re === 'binary' || re === 'b') && lib.typeOf(val) === 'string')<commit_msg>Confirm no bad elements (undefined, null) in an array<commit_after>var sys = require('sys')
  , lib = require('./lib.js')
  ;

function is_int(val) {
  return (!isNaN(val)) && (parseFloat(val) === parseInt(val));
}

// Return the "meat" of a regex, or null if it is not a regex.
function regex_of(val) {
  return lib.typeOf(val) === 'regexp' ? val.toString().slice(1, -1) : null;
}

// Use object creation because I don't like object literal syntax to place functions in a namespace.
var Encoder = function() {
  var self = this;

  this.encode = function(term) {
    var encoder = this[lib.typeOf(term)];
    if(!encoder)
      throw new Error("Do not know how to encode " + lib.typeOf(term) + ': ' + sys.inspect(term));
    return encoder.apply(self, [term]);
  }

  this.number = function(x) {
    return is_int(x) ? this.int(x) : this.float(x);
  }

  this.int = function(x) {
    if(x >= 0 && x < 256)
      return [lib.tags.SMALL_INTEGER, x];
    else
      throw new Error('Unknown integer: ' + x);
  }

  this.raw_array = function(x) {
    // Simple array encoding, without worrying about tagging.
    var result = []
      , encoded = [];

    for(var a = 0; a < x.length; a++) {
      var val = x[a];
      if(!val)
        // TODO: Warning: new Error("Bad array: " + sys.inspect(x));
        continue;
      encoded.push(self.encode(val));
    }

    if(encoded.length) {
      result.push( lib.tags.LIST
                 , lib.uint32(encoded.length)
                 , encoded );
    }

    result.push(lib.tags.NIL);
    return result;
  }

  this.array = function(x) {
    if(x.length !== 2)
      // All arays not of length 2 are innocent.
      return self.raw_array(x);

    // 2-Arrays are used to encode special-cases where Javascript has no syntax to match Erlang.
    var tag = x[0], val = x[1];
    var re = regex_of(tag);

    if(!re)
      // Nope, just an innocent 2-array.
      return self.raw_array(x);

    if((re === 'binary' || re === 'b') && lib.typeOf(val) === 'string')
{'fraction_non_alphanumeric': 0.15992549228312933, 'fraction_numerical': 0.007716870675891432, 'mean_word_length': 2.5698005698005697, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '761533', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1265, 'n_words': 356}