<commit_before>(function() {
  'use strict';

  var PRESENT = function(possible) {
        return typeof possible !== "undefined";
      },
      FLOAT = /^(\-|\+)?([0-9]+(\.[0-9]+)?|Infinity)$/,
      BOOL = function(possible) {
        possible = possible.toLowerCase();
        return (possible === "true" || possible === "false");
      };

  var Builder = function(type, schema, sample, shouldCast) {
    var code = "return ",
        cast = shouldCast ? function(element, index) {
          if (FLOAT.test(element)) {
            return "Number(values[" + index + "]),";
          } else if (BOOL(element)) {
            return "Boolean(values[" + index + "].toLowerCase() === 'true'),";
          } else {
            return "String(values[" + index + "]),";
          }
        } : function(element, index) {
          return "values[" + index + "],";
        },
        _index;

    if (type === "object") {
      code += "{";
      for (_index = 0; _index < schema.length; ++_index) {
        code += '"' + schema[_index] + '": ' + cast(sample[_index], _index);
      }
      code = code.slice(0, -1) + "}";
    } else {
      code += "[";
      for (_index = 0; _index < schema.length; ++_index) {
        code += cast(sample[_index], _index);
      }
      code = code.slice(0, -1) + "]";
    }
    return new Function("values", "cheese", code);
  };

  var CSV = function(data, set) {
    set = PRESENT(set) ? set : {};

    this.options = {
      cast: PRESENT(set.cast) ? set.cast : true,
      line: PRESENT(set.line) ? set.line : "\r\n",
      delimiter: PRESENT(set.delimiter) ? set.delimiter : ",",
      header: PRESENT(set.header) ? set.header : false,
      done: PRESENT(set.done) ? set.done : undefined
    };
    this.data = data;

    return this;
  };

  CSV.prototype.set = function(option, value) {
    this.options[option] = value;
    return this;
  };

  CSV.prototype.encode = function(stream) {
    if (this.data.length === 0) return "";

    var data = this.data,
        response = [],
        delimiter = this.options.delimiter,
        kind = data[0] instanceof Array ? "array" : "object",
        header = this.options.header,
        complete = this.options.done,

        stringify = function(value) {
          return FLOAT.test(value) ? value : '"' + value.replace(/\"/g, '""') + '"';
        },

        sendLine = stream ? function(line) {
          stream(line.join(delimiter));
        } : function(line) {
          response.push(line.join(delimiter));
        },
<commit_msg>handle null values in array of objects<commit_after>(function() {
  'use strict';

  var PRESENT = function(possible) {
        return typeof possible !== "undefined";
      },
      FLOAT = /^(\-|\+)?([0-9]+(\.[0-9]+)?|Infinity)$/,
      BOOL = function(possible) {
        possible = possible.toLowerCase();
        return (possible === "true" || possible === "false");
      };

  var Builder = function(type, schema, sample, shouldCast) {
    var code = "return ",
        cast = shouldCast ? function(element, index) {
          if (FLOAT.test(element)) {
            return "Number(values[" + index + "]),";
          } else if (BOOL(element)) {
            return "Boolean(values[" + index + "].toLowerCase() === 'true'),";
          } else {
            return "String(values[" + index + "]),";
          }
        } : function(element, index) {
          return "values[" + index + "],";
        },
        _index;

    if (type === "object") {
      code += "{";
      for (_index = 0; _index < schema.length; ++_index) {
        code += '"' + schema[_index] + '": ' + cast(sample[_index], _index);
      }
      code = code.slice(0, -1) + "}";
    } else {
      code += "[";
      for (_index = 0; _index < schema.length; ++_index) {
        code += cast(sample[_index], _index);
      }
      code = code.slice(0, -1) + "]";
    }
    return new Function("values", "cheese", code);
  };

  var CSV = function(data, set) {
    set = PRESENT(set) ? set : {};

    this.options = {
      cast: PRESENT(set.cast) ? set.cast : true,
      line: PRESENT(set.line) ? set.line : "\r\n",
      delimiter: PRESENT(set.delimiter) ? set.delimiter : ",",
      header: PRESENT(set.header) ? set.header : false,
      done: PRESENT(set.done) ? set.done : undefined
    };
    this.data = data;

    return this;
  };

  CSV.prototype.set = function(option, value) {
    this.options[option] = value;
    return this;
  };

  CSV.prototype.encode = function(stream) {
    if (this.data.length === 0) return "";

    var data = this.data,
        response = [],
        delimiter = this.options.delimiter,
        kind = data[0] instanceof Array ? "array" : "object",
        header = this.options.header,
        complete = this.options.done,

        stringify = function(value) {
          if (!value) return null;
          return FLOAT.test(value) ? value : '"' + value.replace(/\"/g, '""') + '"';
        },

        sendLine = stream ? function(line) {
          stream(line.join(delimiter));
        } : function(line) {
          response.push(line.join(delimiter));
        },

{'fraction_non_alphanumeric': 0.18324197337509787, 'fraction_numerical': 0.004698512137823023, 'mean_word_length': 2.136279926335175, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5942561', 'n_tokens_mistral': 1632, 'n_tokens_neox': 1587, 'n_words': 340}