<commit_before>var fs = require('fs')
var os = require('os')
var path = require('path')
var tar = require('tar-fs')
var gunzip = require('gunzip-maybe')
var mkdir = require('mkdirp')
var nugget = require('nugget')
var cpr = require('cpr')
var through = require('through2')
var rimraf = require('rimraf')
var yaml = require('yamljs')

var tmpDir = path.join(os.tmpDir(), 'electron-tmp-download')

// var setup = {
//   latest: boolean,
//   version: string,
//   finalDir: string
// }

module.exports = function fetchDocs (setup, callback) {
  var url = 'https://api.github.com/repos/atom/electron/tarball/' + setup.version

  // Create temp directory to fetch tarball into
  mkdir(tmpDir, function (error) {
    if (error) return console.log(error)
    var filename = 'electron.tar.gz'
    var opts = {
      headers: {"user-agent": "Electron"},
      target: filename,
      dir: tmpDir,
      resume: true,
      verbose: true
      }

    nugget(url, opts, function (error) {
      if (error) return console.log(error)
      extractDocs(tmpDir, filename, setup, callback)
    })
  })
}

// Extract just the 'docs' directory from within
// tarball, then prepend each markdown file with
// Jekyll front matter
function extractDocs (tmpDir, filename, setup, callback) {
  var tarball = path.join(tmpDir, filename)
  var newDir = ''

  var extract = tar.extract(tmpDir, {
    ignore: function (name) {
      return name.indexOf('/docs/') === -1
    },
    mapStream: function (fileStream, header) {
      var frontmatter
      if (path.extname(header.name) === '.md') {
        if (path.basename(header.name, '.md') === 'README') {
          console.log("readme", header.name, path.basename(header.name, '.md'))
          var redirectUrl = '/docs/' + setup.version + '/'
          frontmatter = new Buffer('---\nredirect_from: ' + redirectUrl + '\n---\n\n')
        } else {
          var redirectUrl = constructRedirectUrl(header.name, setup.version)
          frontmatter = new Buffer('---\nredirect_from: ' + redirectUrl + '\n---\n\n')
        }
        return fileStream.pipe(frontMatterify(frontmatter))
      }
      return filestream
    }
  })

  extract.on('entry', function extracting (header, stream, next) {
    var extractedElectronDir = header.name.split('/')[0]
    newDir = path.join(tmpDir, extractedElectronDir)
  })

  extract.on('finish', function extracted () {
    var versionDir = path.join(newDir, setup.version)
    var extractedDocsDir = path.join(newDir, 'docs')
    var finalDir = path.join(process.cwd(), setup.finalDir, setup.version)<commit_msg>Better streams for frontmatter & url cleaning<commit_after>var fs = require('fs')
var os = require('os')
var path = require('path')
var tar = require('tar-fs')
var gunzip = require('gunzip-maybe')
var mkdir = require('mkdirp')
var nugget = require('nugget')
var cpr = require('cpr')
var through = require('through2')
var rimraf = require('rimraf')
var yaml = require('yamljs')

var tmpDir = path.join(os.tmpDir(), 'electron-tmp-download')

// var setup = {
//   latest: boolean,
//   version: string,
//   finalDir: string
// }

module.exports = function fetchDocs (setup, callback) {
  var url = 'https://api.github.com/repos/atom/electron/tarball/' + setup.version

  // Create temp directory to fetch tarball into
  mkdir(tmpDir, function (error) {
    if (error) return console.log(error)
    var filename = 'electron.tar.gz'
    var opts = {
      headers: {"user-agent": "Electron"},
      target: filename,
      dir: tmpDir,
      resume: true,
      verbose: true
      }

    nugget(url, opts, function (error) {
      if (error) return console.log(error)
      extractDocs(tmpDir, filename, setup, callback)
    })
  })
}

// Extract just the 'docs' directory from within
// tarball, then prepend each markdown file with
// Jekyll front matter
function extractDocs (tmpDir, filename, setup, callback) {
  var tarball = path.join(tmpDir, filename)
  var newDir = ''

  var extract = tar.extract(tmpDir, {
    ignore: function (name) {
      return name.indexOf('/docs/') === -1
    },
    mapStream: function (fileStream, header) {
      var frontmatter
      if (path.extname(header.name) === '.md') {
        var metadata = constructDocMetadata(header.name, settings.version)
        metadata.source_url = constructSourceUrl(header.name)
        if (path.basename(header.name, '.md') === 'README') {
          metadata.permalink = '/docs/' + settings.version + '/index.html'
          frontmatter = new Buffer('---\n' + yaml.stringify(metadata) + '---\n\n')
          return fileStream.pipe(frontMatterify(frontmatter)).pipe(removeMdUrls())
        }
        frontmatter = new Buffer('---\n' + yaml.stringify(metadata) + '---\n\n')
        return fileStream.pipe(frontMatterify(frontmatter))
      }
      return filestream
    }
  })

  extract.on('entry', function extracting (header, stream, next) {
    var extractedElectronDir = header.name.split('/')[0]
    newDir = path.join(tmpDir, extractedElectronDir)
  })

  extract.on('finish', function extracted () {
    var versionDir = path.join(newDir, setup.version)
    var extractedDocsDir = path.join(newDir, 'docs')
    var finalDir = path.join(process.cwd(), setup.finalDir, setup.version)
{'fraction_non_alphanumeric': 0.15017195261750096, 'fraction_numerical': 0.0011463507833397019, 'mean_word_length': 3.158061953931692, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6846743', 'n_tokens_mistral': 1671, 'n_tokens_neox': 1622, 'n_words': 407}