<commit_before>'use strict'

const debug = require('debug')
const log = debug('exporter')
log.err = debug('exporter:error')
const isIPFS = require('is-ipfs')
const bs58 = require('bs58')
const UnixFS = require('ipfs-unixfs')
const series = require('run-series')
const Readable = require('readable-stream').Readable
const pathj = require('path')
const util = require('util')
const fieldtrip = require('field-trip')

exports = module.exports = Exporter

util.inherits(Exporter, Readable)

function Exporter (hash, dagService, options) {
  if (!(this instanceof Exporter)) {
    return new Exporter(hash, dagService, options)
  }

  // Sanitize hash.
  if (!isIPFS.multihash(hash)) {
    throw new Error('not valid multihash')
  }
  if (Buffer.isBuffer(hash)) {
    hash = bs58.encode(hash)
  }

  Readable.call(this, { objectMode: true })

  this.options = options || {}

  this._read = (n) => {}

  let fileExporter = (node, name, done) => {
    let init

    if (!done) throw new Error('done must be set')

    // Logic to export a single (possibly chunked) unixfs file.
    var rs = new Readable()
    if (node.links.length === 0) {
      const unmarshaledData = UnixFS.unmarshal(node.data)
      init = false
      rs._read = () => {
        if (init) {
          return
        }
        init = true
        rs.push(unmarshaledData.data)
        rs.push(null)
      }
      this.push({ content: rs, path: name })
      done()
    } else {
      init = false
      rs._read = () => {
        if (init) {
          return
        }
        init = true

        const array = node.links.map((link) => {
          return (cb) => {
            dagService.get(link.hash, (err, res) => {
              if (err) {
                return cb(err)
              }
              var unmarshaledData = UnixFS.unmarshal(res.data)<commit_msg>revisions<commit_after>'use strict'

const debug = require('debug')
const log = debug('exporter')
log.err = debug('exporter:error')
const isIPFS = require('is-ipfs')
const bs58 = require('bs58')
const UnixFS = require('ipfs-unixfs')
const series = require('run-series')
const Readable = require('readable-stream').Readable
const pathj = require('path')
const util = require('util')
const fieldtrip = require('field-trip')

exports = module.exports = Exporter

util.inherits(Exporter, Readable)

function Exporter (hash, dagService, options) {
  if (!(this instanceof Exporter)) {
    return new Exporter(hash, dagService, options)
  }

  // Sanitize hash.
  if (!isIPFS.multihash(hash)) {
    throw new Error('not valid multihash')
  }
  if (Buffer.isBuffer(hash)) {
    hash = bs58.encode(hash)
  }

  Readable.call(this, { objectMode: true })

  this.options = options || {}

  this._read = (n) => {}

  let fileExporter = (node, name, done) => {
    let init = false

    if (!done) throw new Error('done must be set')

    // Logic to export a single (possibly chunked) unixfs file.
    var rs = new Readable()
    if (node.links.length === 0) {
      const unmarshaledData = UnixFS.unmarshal(node.data)
      rs._read = () => {
        if (init) {
          return
        }
        init = true
        rs.push(unmarshaledData.data)
        rs.push(null)
      }
      this.push({ content: rs, path: name })
      done()
    } else {
      rs._read = () => {
        if (init) {
          return
        }
        init = true

        const array = node.links.map((link) => {
          return (cb) => {
            dagService.get(link.hash, (err, res) => {
              if (err) {
                return cb(err)
              }
              var unmarshaledData = UnixFS.unmarshal(res.data)
{'fraction_non_alphanumeric': 0.1403750689464975, 'fraction_numerical': 0.003861003861003861, 'mean_word_length': 2.3992502343017805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2590140', 'n_tokens_mistral': 1193, 'n_tokens_neox': 1136, 'n_words': 290}