<commit_before>
// copy - copy source directory and its subtree to target directory

var fstream = require('fstream')
  , resolve = require('path').resolve
  , Readable = require('stream').Readable

module.exports = function (source, target, callback) {
  var reader = fstream.Reader({ path: source, filter: filter })
    , writer = fstream.Writer({ path: target, type: 'Directory' })
    , stream = new Readable()

  reader.on('error', function (err) {
    stream.emit('error', err)
  })

  reader.on('entry', function (entry) {
    push(entry)
  })

  writer.on('error', function (err) {
    stream.emit('error', err)
  })

  writer.on('end', function () {
    stream.emit('end')
    if (callback) callback()
  })

  function push (entry) {
    if (entry.type === 'File') {
      // TODO: correct output path
      stream.push(entry.basename + '\n')
    } else {
      entry.on('entry', push)
    }
  }

  stream._read = function (size, callback) {
    // well ...
  }

  reader.pipe(writer)
<commit_msg>Push correct paths of copies<commit_after>
// copy - copy source directory and its subtree to target directory

var fstream = require('fstream')
  , resolve = require('path').resolve
  , Readable = require('stream').Readable
  , path = require('path')

module.exports = function (source, target, callback) {
  var reader = fstream.Reader({ path: source, filter: filter })
    , writer = fstream.Writer({ path: target, type: 'Directory' })
    , stream = new Readable()

  reader.on('error', function (err) {
    stream.emit('error', err)
  })

  reader.on('entry', function (entry) {
    push(entry)
  })

  writer.on('error', function (err) {
    stream.emit('error', err)
  })

  writer.on('end', function () {
    stream.emit('end')
    if (callback) callback()
  })

  function push (entry) {
    if (entry.type === 'File') {
      stream.push(path.join(target, entry.path.split(source)[1]) + '\n')
    } else {
      entry.on('entry', push)
    }
  }

  stream._read = function (size, callback) {
    // well ...
  }

  reader.pipe(writer)

{'fraction_non_alphanumeric': 0.1696733300828864, 'fraction_numerical': 0.00048756704046806434, 'mean_word_length': 3.0795228628230618, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2424672', 'n_tokens_mistral': 660, 'n_tokens_neox': 645, 'n_words': 163}