<commit_before>// var flatten = require('flat')
var fs = require('fs')
var dsv = require('d3-dsv')
var dbf = require('dbf')
var shapefileDbf = require('shapefile/dbf')
var path = require('path')
var queue = require('queue-async')
var _ = require('underscore')
var chalk = require('chalk')
var json_parser = require('parse-json') // This library has better error reporting of malformed json.
var yaml_parser = require('js-yaml')
var mkdirp = require('mkdirp')
var archieml = require('archieml')

var formatters = {
  json: function (file, writeOptions) {
    writeOptions = writeOptions || {}
    return JSON.stringify(file, writeOptions.replacer, writeOptions.space)
  },
  csv: function (file, writeOptions) {
    file = formattingPreflight(file, 'csv')
    try {
      return dsv.csvFormat(file, writeOptions)
    } catch (err) {
      reporters.parseError('csv')
    }
  },
  tsv: function (file, writeOptions) {
    file = formattingPreflight(file, 'tsv')
    try {
      return dsv.tsvFormat(file, writeOptions)
    } catch (err) {
      reporters.parseError('tsv')
    }
  },
  psv: function (file, writeOptions) {
    file = formattingPreflight(file, 'psv')
    try {
      return dsv.dsvFormat('|').format(file, writeOptions)
    } catch (err) {
      reporters.parseError('psv')
    }
  },
  yaml: function (file, writeOptions) {
    // TK Write options
    return yaml_parser.dump(file)
  },
  txt: _.identity,
  dbf: function (file, writeOptions) {
    writeOptions = writeOptions || {}
    function toBuffer (ab) {
      var buffer = new Buffer(ab.byteLength)
      var view = new Uint8Array(ab)
      for (var i = 0; i < buffer.length; ++i) {
        buffer[i] = view[i]
      }
      return buffer
    }
    var buf = dbf.structure(file)
    return toBuffer(buf.buffer)
  }
}

// Each parser must return the parsed data
// For legacy support, they can also have a `.parse` method
var parsers = {
  json: function (str, readOptions) {
    readOptions = readOptions || {}
    return json_parser(str, readOptions.reviver, readOptions.filename)
  },
  csv: dsv.csvParse,
  tsv: dsv.tsvParse,
  psv: dsv.dsvFormat('|').parse,
  txt: _.identity, // Because we pass `utf-8` in to `readFile` and `readFileSync`, our data object is going to be a string, so we can simply return it,
  yaml: yaml_parser.load,
  aml: archieml.load
}

// Alias `.yml` to `.yaml`
parsers.yml = parsers.yaml

var reporters = {}

reporters.warn = function (msg) {
  console.log(chalk.gray('[indian-ocean]') + ' ' + chalk.yellow('Warning:', msg))
}

reporters.parseError = function (format) {
  throw new Error(chalk.red('[indian-ocean] Error converting your data to ' + chalk.bold(format) + '.') + '\n\n' + chalk.cyan('Your data most likely contains objects or lists. Object values can only be strings for this format. Please convert before writing to file.\n'))
}

reporters.notListError = function (format) {
  throw new Error(chalk.red('[indian-ocean] You passed in an object but converting to ' + chalk.bold(format) + ' requires a list of objects.') + chalk.cyan('\nIf you would like to write a one-row csv, put your object in a list like so: `' + chalk.bold('[data]') + '`\n'))
}

/**
  * Various utility functions.
  * @namespace
*/
var helpers = {}

/**
 * Given a `fileName` return its file extension. Used internally by `.discernPaser` and `.discernFileFormatter`.
 * @param {String} fileName the name of the file
 * @returns {String} the file's extension
 *
 * @example
 * var format = io.discernFormat('path/to/data.csv')
 * console.log(format) // 'csv'
 */<commit_msg>add better undefined file error for json<commit_after>// var flatten = require('flat')
var fs = require('fs')
var dsv = require('d3-dsv')
var dbf = require('dbf')
var shapefileDbf = require('shapefile/dbf')
var path = require('path')
var queue = require('queue-async')
var _ = require('underscore')
var chalk = require('chalk')
var json_parser = require('parse-json') // This library has better error reporting of malformed json.
var yaml_parser = require('js-yaml')
var mkdirp = require('mkdirp')
var archieml = require('archieml')

var formatters = {
  json: function (file, writeOptions) {
    writeOptions = writeOptions || {}
    return JSON.stringify(file, writeOptions.replacer, writeOptions.space)
  },
  csv: function (file, writeOptions) {
    file = formattingPreflight(file, 'csv')
    try {
      return dsv.csvFormat(file, writeOptions)
    } catch (err) {
      reporters.parseError('csv')
    }
  },
  tsv: function (file, writeOptions) {
    file = formattingPreflight(file, 'tsv')
    try {
      return dsv.tsvFormat(file, writeOptions)
    } catch (err) {
      reporters.parseError('tsv')
    }
  },
  psv: function (file, writeOptions) {
    file = formattingPreflight(file, 'psv')
    try {
      return dsv.dsvFormat('|').format(file, writeOptions)
    } catch (err) {
      reporters.parseError('psv')
    }
  },
  yaml: function (file, writeOptions) {
    // TK Write options
    return yaml_parser.dump(file)
  },
  txt: _.identity,
  dbf: function (file, writeOptions) {
    writeOptions = writeOptions || {}
    function toBuffer (ab) {
      var buffer = new Buffer(ab.byteLength)
      var view = new Uint8Array(ab)
      for (var i = 0; i < buffer.length; ++i) {
        buffer[i] = view[i]
      }
      return buffer
    }
    var buf = dbf.structure(file)
    return toBuffer(buf.buffer)
  }
}

// Each parser must return the parsed data
// For legacy support, they can also have a `.parse` method
var parsers = {
  json: function (str, readOptions) {
    readOptions = readOptions || {}
    return json_parser(str, readOptions.reviver, readOptions.filename)
  },
  csv: dsv.csvParse,
  tsv: dsv.tsvParse,
  psv: dsv.dsvFormat('|').parse,
  txt: _.identity, // Because we pass `utf-8` in to `readFile` and `readFileSync`, our data object is going to be a string, so we can simply return it,
  yaml: yaml_parser.load,
  aml: archieml.load
}

// Alias `.yml` to `.yaml`
parsers.yml = parsers.yaml

var reporters = {}

reporters.warn = function (msg) {
  console.log(chalk.gray('[indian-ocean]') + ' ' + chalk.yellow('Warning:', msg))
}

reporters.writeError = function (format, msg) {
  throw new Error(chalk.red('[indian-ocean] Error writing your data to ' + chalk.bold(format) + '.') + '\n\n' + msg)
}

reporters.parseError = function (format) {
  throw new Error(chalk.red('[indian-ocean] Error converting your data to ' + chalk.bold(format) + '.') + '\n\n' + chalk.cyan('Your data most likely contains objects or lists. Object values can only be strings for this format. Please convert before writing to file.\n'))
}

reporters.notListError = function (format) {
  throw new Error(chalk.red('[indian-ocean] You passed in an object but converting to ' + chalk.bold(format) + ' requires a list of objects.') + chalk.cyan('\nIf you would like to write a one-row csv, put your object in a list like so: `' + chalk.bold('[data]') + '`\n'))
}

/**
  * Various utility functions.
  * @namespace
*/
var helpers = {}

/**
 * Given a `fileName` return its file extension. Used internally by `.discernPaser` and `.discernFileFormatter`.
 * @param {String} fileName the name of the file
 * @returns {String} the file's extension
 *
 * @example
 * var format = io.discernFormat('path/to/data.csv')
 * console.log(format) // 'csv'
 */
{'fraction_non_alphanumeric': 0.14113949993168465, 'fraction_numerical': 0.001093045498018855, 'mean_word_length': 3.5325077399380804, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3161162', 'n_tokens_mistral': 2457, 'n_tokens_neox': 2383, 'n_words': 697}