<commit_before>#!/usr/bin/env node

var hyperquest = require('hyperquest')
var concat = require('concat-stream')
var split  = require('split')
var thru = require('through2')
var fs = require('fs')

var url = 'https://api.github.com/repos/iojs/io.js/contents'
var dirs = [
  '/test/parallel',
  '/test/pummel'
]

var httpOpts = {
  headers: {
    'User-Agent': null,

    // auth if github rate-limits you...
    // 'Authorization': 'Basic ' + Buffer('username:password').toString('base64'),
  }
}

dirs.forEach(function(dir) {
  var req = hyperquest(url + dir, httpOpts)
  req.pipe(concat(function(data) {
    if (req.response.statusCode !== 200) {
      throw new Error(url + dir + ': ' + data.toString())
    }

    downloadBufferTests(dir, JSON.parse(data));
  }))
})

function downloadBufferTests(dir, files) {
  files.forEach(function(file) {
    if (!/test-buffer.*/.test(file.name)) return

    hyperquest(file.download_url, httpOpts)
      .pipe(split())
      .pipe(testfixer(file.name))
      .pipe(fs.createWriteStream(__dirname + '/../test/node-' + file.name))
  })
}

function testfixer(filename) {
  var firstline = true

  return thru(function(line, enc, cb) {
    line = line.toString()

    if (firstline) {

      // require buffer explicitly and wrap in tape test
      line = 'var Buffer = require(\'../\').Buffer\n' +
             'var test = require(\'tape\')\n' +
             'if (process.env.OBJECT_IMPL) Buffer.TYPED_ARRAY_SUPPORT = false\n' +
             'test(\'' + filename + '\', function(t) {\n' + line

      firstline = false
    }

    // use tape not assert
    if (/^var assert/.test(line)) {
      line = line.replace(/(^var assert.*)/, '// $1')
    }
    else {

      // use "ok" instead of shortcut
      line = line.replace(/assert\(/, 'assert.ok(')

      // use tape
      line = line.replace(/assert/, 't')
    }

    // comment out require('common')
    line = line.replace(/^(var common = require.*)/, '// $1')

    // require browser buffer
    line = line.replace(/(.*)require\('buffer'\)(.*)/, '$1require(\'../\')$2')

    // smalloc is only used for kMaxLength
    line = line.replace(/require\('smalloc'\)/, '{ kMaxLength: 0x3FFFFFFF }')

    // comment out console logs
    line = line.replace(/(.*console\..*)/, '// $1')

    // tape's equal can't compare strings and buffers
    line = line.replace(/t.equal\(buf.slice\(([^\)]*)\), /, 't.equal(buf.slice($1).toString(), ')

    // we can't reliably test typed array max-sizes in the browser
    if (filename === 'test-buffer-big.js') {
      line = line.replace(/(.*new Int8Array.*RangeError.*)/, '// $1')
      line = line.replace(/(.*new ArrayBuffer.*RangeError.*)/, '// $1')
      line = line.replace(/(.*new Float64Array.*RangeError.*)/, '// $1')
    }

    // https://github.com/iojs/io.js/blob/v0.12/test/parallel/test-buffer.js#L38
    // we can't run this because we need to support
    // browsers that don't have typed arrays
    if (filename === 'test-buffer.js') {
      line = line.replace(/b\[0\] = -1;/, 'b[0] = 255;')
    }

    // https://github.com/iojs/io.js/blob/v0.12/test/parallel/test-buffer.js#L1138
    // unfortunately we can't run this as it touches
    // node streams which do an instanceof check
    if (filename === 'test-buffer.js') {
      line = line.replace(/(crypto.createHash.*\))/, '1 /*$1*/')
    }

    cb(null, line + '\n')
  }, function(cb) {

    // close tape wrapper
    this.push('\nt.end()\n})')

    cb()
  })
}<commit_msg>do not transform node tests to use tape<commit_after>#!/usr/bin/env node

var hyperquest = require('hyperquest')
var concat = require('concat-stream')
var split  = require('split')
var thru = require('through2')
var fs = require('fs')

var url = 'https://api.github.com/repos/iojs/io.js/contents'
var dirs = [
  '/test/parallel',
  '/test/pummel'
]

var httpOpts = {
  headers: {
    'User-Agent': null,

    // auth if github rate-limits you...
    // 'Authorization': 'Basic ' + Buffer('username:password').toString('base64'),
  }
}

dirs.forEach(function(dir) {
  var req = hyperquest(url + dir, httpOpts)
  req.pipe(concat(function(data) {
    if (req.response.statusCode !== 200) {
      throw new Error(url + dir + ': ' + data.toString())
    }

    downloadBufferTests(dir, JSON.parse(data));
  }))
})

function downloadBufferTests(dir, files) {
  files.forEach(function(file) {
    if (!/test-buffer.*/.test(file.name)) return

    hyperquest(file.download_url, httpOpts)
      .pipe(split())
      .pipe(testfixer(file.name))
      .pipe(fs.createWriteStream(__dirname + '/../test/node-' + file.name))
  })
}

function testfixer(filename) {
  var firstline = true

  return thru(function(line, enc, cb) {
    line = line.toString()

    if (firstline) {

      // require buffer explicitly
      line = 'var Buffer = require(\'../\').Buffer\n' +
             'if (process.env.OBJECT_IMPL) Buffer.TYPED_ARRAY_SUPPORT = false\n' + line

      firstline = false
    }

    // comment out require('common')
    line = line.replace(/^(var common = require.*)/, '// $1')

    // require browser buffer
    line = line.replace(/(.*)require\('buffer'\)(.*)/, '$1require(\'../\')$2')

    // smalloc is only used for kMaxLength
    line = line.replace(/require\('smalloc'\)/, '{ kMaxLength: 0x3FFFFFFF }')

    // comment out console logs
    line = line.replace(/(.*console\..*)/, '// $1')

    // we can't reliably test typed array max-sizes in the browser
    if (filename === 'test-buffer-big.js') {
      line = line.replace(/(.*new Int8Array.*RangeError.*)/, '// $1')
      line = line.replace(/(.*new ArrayBuffer.*RangeError.*)/, '// $1')
      line = line.replace(/(.*new Float64Array.*RangeError.*)/, '// $1')
    }

    // https://github.com/iojs/io.js/blob/v0.12/test/parallel/test-buffer.js#L38
    // we can't run this because we need to support
    // browsers that don't have typed arrays
    if (filename === 'test-buffer.js') {
      line = line.replace(/b\[0\] = -1;/, 'b[0] = 255;')
    }

    // https://github.com/iojs/io.js/blob/v0.12/test/parallel/test-buffer.js#L1138
    // unfortunately we can't run this as it touches
    // node streams which do an instanceof check
    if (filename === 'test-buffer.js') {
      line = line.replace(/(crypto.createHash.*\))/, '1 /*$1*/')
    }

    cb(null, line + '\n')
  })
}
{'fraction_non_alphanumeric': 0.19572446555819478, 'fraction_numerical': 0.012351543942992874, 'mean_word_length': 2.6257175660160734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5493815', 'n_tokens_mistral': 2205, 'n_tokens_neox': 2109, 'n_words': 462}