<commit_before>var fs = require('fs')
var dsv = require('d3-dsv')
var dbf = require('dbf')
var shapefileDbf = require('shapefile/dbf')
var path = require('path')
var queue = require('queue-async')
var _ = require('underscore')
var chalk = require('chalk')
var json_parser = require('parse-json') // This library has better error reporting of malformed json.
var yaml_parser = require('js-yaml')
var mkdirp = require('mkdirp')

var formatters = {
  json: function (file) {
    return JSON.stringify(file)
  },
  csv: function (file) {
    try {
      return dsv.csv.format(file)
    } catch (err) {
      reporters.parseError('csv')
    }
  },
  tsv: function (file) {
    try {
      return dsv.tsv.format(file)
    } catch (err) {
      reporters.parseError('tsv')
    }
  },
  psv: function (file) {
    try {
      return dsv.dsv('|').format(file)
    } catch (err) {
      reporters.parseError('psv')
    }
  },
  yaml: function (file) {
    return yaml_parser.dump(file)
  },
  txt: function (file) {
    return file
  },
  dbf: function (file) {
    function toBuffer (ab) {
      var buffer = new Buffer(ab.byteLength)
      var view = new Uint8Array(ab)
      for (var i = 0; i < buffer.length; ++i) {
        buffer[i] = view[i]
      }
      return buffer
    }
    var buf = dbf.structure(file)
    return toBuffer(buf.buffer)
  }
}

// Each parser needs a `.parse` method
// dsv parsers come with one for free
var parsers = {
  json: {
    parse: json_parser
  },
  csv: dsv.csv,
  tsv: dsv.tsv,
  psv: dsv.dsv('|'),
  txt: {
    parse: function (data) {
      // Because we pass `utf-8` in to `readFile` and `readFileSync`, our data object is going to be a string, so we can simply return it
      return data
    }
  },
  yaml: {
    parse: function (data) {
      return yaml_parser.load(data)
    }
  }
}

// Alias `.yml` to `.yaml`
parsers.yml = parsers.yaml

var reporters = {}

reporters.warn = function (msg) {
  console.log(chalk.yellow('[indian-ocean] Warning:', msg))
}

reporters.parseError = function (format) {
  throw new Error(chalk.red('Error converting your data to ' + chalk.bold(format) + '.') + '\n\n' + chalk.cyan('Your data most likely contains objects or lists. Object values can only be strings for this format. Please convert before writing to file.\n'))
}

/** @namespace */
var helpers = {}

/**
 * Given a `fileName` return its file extension. Used internally by `.discernPaser` and `.discernFileFormatter`.
 * @param {String} fileName the name of the file
 * @returns {String} the file's extension
 *
 * @example
 * var format = io.discernFormat('path/to/data.csv')
 * console.log(format) // 'csv'
 */
helpers.discernFormat = function (fileName) {
  var extension = path.extname(fileName)
  if (extension === '') return false

  var formatName = extension.slice(1)
  // If it ends in json such as `topojson` or `geojson`, call it json.
  if (formatName.substr(formatName.length - 4) === 'json') {
    formatName = 'json'
  }
  return formatName
}

/**
 * Given a `fileName`, optionally a delimiter, return a parser that can read that file as json. Parses as text if format not supported. Used internally by `.readData` and `.readDataSync`.
 * @param {String} fileName the name of the file
 * @param {Object} options optionally can take a delimiter value
 * @returns {Object} a parser that can read the file
 *
 * @example
 * var parser = io.discernParser('path/to/data.csv');
 * var json = parser('path/to/data.csv');

 * var parser = io.discernParser('path/to/data.usv', {delimiter: '_'});
 * var json = parser('path/to/data.usv');
 */
helpers.discernParser = function (fileName, opts_) {
  if (opts_ && opts_.delimiter) {
    return dsv.dsv(opts_.delimiter)
  }
  var format = helpers.discernFormat(fileName)
  var parser = parsers[format]
  // If we don't have a parser for this format, notify the user and return as text
  if (typeof parser === 'undefined') {
    console.log(chalk.cyan('[indian-ocean] Notice:'), '"' + chalk.bold(format) + '" extension not supported on file: "' + chalk.bold(fileName) + '". Parsing as plain text...')
    parser = parsers['txt']
  }
  return parser
}

/**
 * Returns a formatter that will format json data to file type specified by the extension in `fileName`. Used internally by `.writeData` and `.writeDataSync`.
 * @param {String} fileName the name of the file
 * @returns {Object} a formatter that can write the file
 *
 * @example
 * var formatter = io.discernFileFormatter('path/to/data.tsv');
 * var csv = formatter(json);
 */
helpers.discernFileFormatter = function (fileName) {
  var format = helpers.discernFormat(fileName)
  var formatter = formatters[format]
  // If we don't have a parser for this format, notify the user and return as text
  if (typeof formatter === 'undefined') {
    console.log(chalk.cyan('[indian-ocean] Notice:'), '"' + chalk.bold(format) + '" extension not supported on file: "' + chalk.bold(fileName) + '". Writing out as plain text...')
    formatter = formatter['txt']
  }
  return formatter
}

/**
 * Asynchronously test whether a file exists or not by using `fs.access` modified from https://github.com/nodejs/io.js/issues/1592#issuecomment-98392899.
 * @param {String} fileName the name of the file
 * @param {Function} callback has the signature `(err, exists)`
 *
 * @example
 * var exists = io.exists('path/to/data.tsv', function (err, exists) {
 *   console.log(exists) // `true` if the file exists, `false` if not.
 * })
 *
 */
helpers.exists = function (filename, cb) {
  fs.access(function (err) {
    cb(err, err !== null)
  })
}

/**<commit_msg>fix exists fn<commit_after>var fs = require('fs')
var dsv = require('d3-dsv')
var dbf = require('dbf')
var shapefileDbf = require('shapefile/dbf')
var path = require('path')
var queue = require('queue-async')
var _ = require('underscore')
var chalk = require('chalk')
var json_parser = require('parse-json') // This library has better error reporting of malformed json.
var yaml_parser = require('js-yaml')
var mkdirp = require('mkdirp')

var formatters = {
  json: function (file) {
    return JSON.stringify(file)
  },
  csv: function (file) {
    try {
      return dsv.csv.format(file)
    } catch (err) {
      reporters.parseError('csv')
    }
  },
  tsv: function (file) {
    try {
      return dsv.tsv.format(file)
    } catch (err) {
      reporters.parseError('tsv')
    }
  },
  psv: function (file) {
    try {
      return dsv.dsv('|').format(file)
    } catch (err) {
      reporters.parseError('psv')
    }
  },
  yaml: function (file) {
    return yaml_parser.dump(file)
  },
  txt: function (file) {
    return file
  },
  dbf: function (file) {
    function toBuffer (ab) {
      var buffer = new Buffer(ab.byteLength)
      var view = new Uint8Array(ab)
      for (var i = 0; i < buffer.length; ++i) {
        buffer[i] = view[i]
      }
      return buffer
    }
    var buf = dbf.structure(file)
    return toBuffer(buf.buffer)
  }
}

// Each parser needs a `.parse` method
// dsv parsers come with one for free
var parsers = {
  json: {
    parse: json_parser
  },
  csv: dsv.csv,
  tsv: dsv.tsv,
  psv: dsv.dsv('|'),
  txt: {
    parse: function (data) {
      // Because we pass `utf-8` in to `readFile` and `readFileSync`, our data object is going to be a string, so we can simply return it
      return data
    }
  },
  yaml: {
    parse: function (data) {
      return yaml_parser.load(data)
    }
  }
}

// Alias `.yml` to `.yaml`
parsers.yml = parsers.yaml

var reporters = {}

reporters.warn = function (msg) {
  console.log(chalk.yellow('[indian-ocean] Warning:', msg))
}

reporters.parseError = function (format) {
  throw new Error(chalk.red('Error converting your data to ' + chalk.bold(format) + '.') + '\n\n' + chalk.cyan('Your data most likely contains objects or lists. Object values can only be strings for this format. Please convert before writing to file.\n'))
}

/** @namespace */
var helpers = {}

/**
 * Given a `fileName` return its file extension. Used internally by `.discernPaser` and `.discernFileFormatter`.
 * @param {String} fileName the name of the file
 * @returns {String} the file's extension
 *
 * @example
 * var format = io.discernFormat('path/to/data.csv')
 * console.log(format) // 'csv'
 */
helpers.discernFormat = function (fileName) {
  var extension = path.extname(fileName)
  if (extension === '') return false

  var formatName = extension.slice(1)
  // If it ends in json such as `topojson` or `geojson`, call it json.
  if (formatName.substr(formatName.length - 4) === 'json') {
    formatName = 'json'
  }
  return formatName
}

/**
 * Given a `fileName`, optionally a delimiter, return a parser that can read that file as json. Parses as text if format not supported. Used internally by `.readData` and `.readDataSync`.
 * @param {String} fileName the name of the file
 * @param {Object} options optionally can take a delimiter value
 * @returns {Object} a parser that can read the file
 *
 * @example
 * var parser = io.discernParser('path/to/data.csv');
 * var json = parser('path/to/data.csv');

 * var parser = io.discernParser('path/to/data.usv', {delimiter: '_'});
 * var json = parser('path/to/data.usv');
 */
helpers.discernParser = function (fileName, opts_) {
  if (opts_ && opts_.delimiter) {
    return dsv.dsv(opts_.delimiter)
  }
  var format = helpers.discernFormat(fileName)
  var parser = parsers[format]
  // If we don't have a parser for this format, notify the user and return as text
  if (typeof parser === 'undefined') {
    console.log(chalk.cyan('[indian-ocean] Notice:'), '"' + chalk.bold(format) + '" extension not supported on file: "' + chalk.bold(fileName) + '". Parsing as plain text...')
    parser = parsers['txt']
  }
  return parser
}

/**
 * Returns a formatter that will format json data to file type specified by the extension in `fileName`. Used internally by `.writeData` and `.writeDataSync`.
 * @param {String} fileName the name of the file
 * @returns {Object} a formatter that can write the file
 *
 * @example
 * var formatter = io.discernFileFormatter('path/to/data.tsv');
 * var csv = formatter(json);
 */
helpers.discernFileFormatter = function (fileName) {
  var format = helpers.discernFormat(fileName)
  var formatter = formatters[format]
  // If we don't have a parser for this format, notify the user and return as text
  if (typeof formatter === 'undefined') {
    console.log(chalk.cyan('[indian-ocean] Notice:'), '"' + chalk.bold(format) + '" extension not supported on file: "' + chalk.bold(fileName) + '". Writing out as plain text...')
    formatter = formatter['txt']
  }
  return formatter
}

/**
 * Asynchronously test whether a file exists or not by using `fs.access` modified from https://github.com/nodejs/io.js/issues/1592#issuecomment-98392899.
 * @param {String} fileName the name of the file
 * @param {Function} callback has the signature `(err, exists)`
 *
 * @example
 * var exists = io.exists('path/to/data.tsv', function (err, exists) {
 *   console.log(exists) // `true` if the file exists, `false` if not.
 * })
 *
 */
helpers.exists = function (filename, cb) {
  fs.access(filename, function (err) {
    var exists
    if (err && err.code == 'ENOENT') {
      exists = false
      err = null
    } else if (!err) {
      exists = true
    }
    cb(err, exists)
  })
}

/**
{'fraction_non_alphanumeric': 0.14219980532696222, 'fraction_numerical': 0.003185558800106185, 'mean_word_length': 3.312094620373903, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6905670', 'n_tokens_mistral': 3767, 'n_tokens_neox': 3686, 'n_words': 1143}