<commit_before>'use strict'

const bs58 = require('bs58')
const multipart = require('ipfs-multipart')
const debug = require('debug')
const tar = require('tar-stream')
const log = debug('http-api:files')
log.error = debug('http-api:files:error')
const pull = require('pull-stream')
const toStream = require('pull-stream-to-stream')
const toPull = require('stream-to-pull-stream')
const pushable = require('pull-pushable')
const EOL = require('os').EOL

exports = module.exports

// common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args`
exports.parseKey = (request, reply) => {
  if (!request.query.arg) {
    return reply("Argument 'key' is required").code(400).takeover()
  }

  try {
    return reply({
      key: new Buffer(bs58.decode(request.query.arg))
    })
  } catch (err) {
    log.error(err)
    return reply({
      Message: 'invalid ipfs ref path',
      Code: 0
    }).code(500).takeover()
  }
}

exports.cat = {
  // uses common parseKey method that returns a `key`
  parseArgs: exports.parseKey,

  // main route handler which is called after the above `parseArgs`, but only if the args were valid
  handler: (request, reply) => {
    const key = request.pre.args.key
    const ipfs = request.server.app.ipfs

    ipfs.files.cat(key, (err, stream) => {
      if (err) {
        log.error(err)
        return reply({
          Message: 'Failed to cat file: ' + err,
          Code: 0
        }).code(500)
      }

      // hapi is not very clever and throws if no
      // - _read method
      // - _readableState object
      // are there :(
      stream._read = () => {}
      stream._readableState = {}
      return reply(stream).header('X-Stream-Output', '1')
    })
  }
}

exports.get = {
  // uses common parseKey method that returns a `key`
  parseArgs: exports.parseKey,

  // main route handler which is called after the above `parseArgs`, but only if the args were valid
  handler: (request, reply) => {
    const key = request.pre.args.key
    const ipfs = request.server.app.ipfs
    const pack = tar.pack()

    ipfs.files.getPull(key, (err, stream) => {
      if (err) return handleError(err)

      pull(
        stream,
        pull.asyncMap((file, cb) => {
          const header = {name: file.path}

          if (!file.content) {
            header.type = 'directory'
            pack.entry(header)
            cb()
          } else {
            header.size = file.size
            toStream.source(file.content)
              .pipe(pack.entry(header, cb))
          }
        }),
        pull.onEnd((err) => {
          if (err) return handleError(err)

          pack.finalize()
          reply(pack).header('X-Stream-Output', '1')
        })
      )
    })

    function handleError (err) {
      log.error(err)

      reply({
        Message: 'Failed to get file: ' + err,
        Code: 0
      }).code(500)
    }
  }
}

exports.add = {
  handler: (request, reply) => {
    if (!request.payload) {
      return reply('Array, Buffer, or String is required.').code(400).takeover()
    }

    const ipfs = request.server.app.ipfs
    // TODO: make pull-multipart
    const parser = multipart.reqParser(request.payload)
    let filesParsed = false
<commit_msg>fix(http): get handler reads the stream<commit_after>'use strict'

const bs58 = require('bs58')
const multipart = require('ipfs-multipart')
const debug = require('debug')
const tar = require('tar-stream')
const log = debug('http-api:files')
log.error = debug('http-api:files:error')
const pull = require('pull-stream')
const toStream = require('pull-stream-to-stream')
const toPull = require('stream-to-pull-stream')
const pushable = require('pull-pushable')
const EOL = require('os').EOL

exports = module.exports

// common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args`
exports.parseKey = (request, reply) => {
  if (!request.query.arg) {
    return reply("Argument 'key' is required").code(400).takeover()
  }

  try {
    return reply({
      key: new Buffer(bs58.decode(request.query.arg))
    })
  } catch (err) {
    log.error(err)
    return reply({
      Message: 'invalid ipfs ref path',
      Code: 0
    }).code(500).takeover()
  }
}

exports.cat = {
  // uses common parseKey method that returns a `key`
  parseArgs: exports.parseKey,

  // main route handler which is called after the above `parseArgs`, but only if the args were valid
  handler: (request, reply) => {
    const key = request.pre.args.key
    const ipfs = request.server.app.ipfs

    ipfs.files.cat(key, (err, stream) => {
      if (err) {
        log.error(err)
        return reply({
          Message: 'Failed to cat file: ' + err,
          Code: 0
        }).code(500)
      }

      // hapi is not very clever and throws if no
      // - _read method
      // - _readableState object
      // are there :(
      stream._read = () => {}
      stream._readableState = {}
      return reply(stream).header('X-Stream-Output', '1')
    })
  }
}

exports.get = {
  // uses common parseKey method that returns a `key`
  parseArgs: exports.parseKey,

  // main route handler which is called after the above `parseArgs`, but only if the args were valid
  handler: (request, reply) => {
    const key = request.pre.args.key
    const ipfs = request.server.app.ipfs
    const pack = tar.pack()

    ipfs.files.getPull(key, (err, stream) => {
      if (err) return handleError(err)

      pull(
        stream,
        pull.asyncMap((file, cb) => {
          const header = {name: file.path}

          if (!file.content) {
            header.type = 'directory'
            pack.entry(header)
            cb()
          } else {
            header.size = file.size
            toStream.source(file.content)
              .pipe(pack.entry(header, cb))
          }
        }),
        pull.onEnd((err) => {
          if (err) return handleError(err)

          pack.finalize()
        })
      )

      // the reply must read the tar stream,
      // to pull values through
      reply(pack).header('X-Stream-Output', '1')
    })

    function handleError (err) {
      log.error(err)

      reply({
        Message: 'Failed to get file: ' + err,
        Code: 0
      }).code(500)
    }
  }
}

exports.add = {
  handler: (request, reply) => {
    if (!request.payload) {
      return reply('Array, Buffer, or String is required.').code(400).takeover()
    }

    const ipfs = request.server.app.ipfs
    // TODO: make pull-multipart
    const parser = multipart.reqParser(request.payload)
    let filesParsed = false

{'fraction_non_alphanumeric': 0.13657056145675264, 'fraction_numerical': 0.007890743550834597, 'mean_word_length': 2.335526315789474, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '102924', 'n_tokens_mistral': 2076, 'n_tokens_neox': 2006, 'n_words': 554}