<commit_before>'use strict';
/*jshint asi: true */

var test      =  require('tap').test
  , xtend     =  require('xtend')
  , level     =  require('level-test')
  , sublevel  =  require('level-sublevel')
  , dump      =  require('level-dump')
  , getLogins =  require('../lib/get-github-logins')
  , npm       =  require('valuepack-core/mine/namespaces').npm
  , sublevels = require('valuepack-core/mine/sublevels');

function toPut (o, valEnc) {
  return xtend(o, { type: 'put' }); 
}

function getPackagesBatch () {
  // each file contains subset of each user's packages
  return require('./fixtures/npm-packages-dominictarr.json')
    .concat(require('./fixtures/npm-packages-raynos.json'))
    .concat(require('./fixtures/npm-packages-substack.json'))
    .concat(require('./fixtures/npm-packages-thlorenz.json'))
    .concat(require('./fixtures/npm-packages-tj.json'))
    .map(toPut)
}

function setup (cb) {
  // contains metadata of these users
  var users = require('./fixtures/npm-users-tj-thlorenz-substack-dominictarr-raynos.json')

  // contains byOwner indexes for all batched packages
    , owners = require('./fixtures/byOwner-tj-thlorenz-substack-dominictarr-raynos.json')
    , usersBatch = users.map(toPut)
    , byOwnerBatch = owners.map(toPut)
    , packagesBatch = getPackagesBatch()

  var db = sublevel(level({ mem: true })(null, { valueEncoding: 'json' }))

    // owner values are utf8 encoded, but since we are reading this in json format, we need to adjust while
    // entering data, otherwise all values are surrounded by ""s
  var byOwner = db.sublevel(npm.byOwner, { valueEncoding: 'json' })

  var subnpm      =  sublevels(db).npm
    , npmPackages =  subnpm.packages
    , npmUsers    =  subnpm.users


  npmPackages.batch(packagesBatch, function (err) {
    if (err) return cb(err);
    byOwner.batch(byOwnerBatch, function (err) {
      if (err) return cb(err);
      npmUsers.batch(usersBatch, function (err) {
        if (err) return cb(err);
        cb(null, db);
      })
    })
  })
}<commit_msg>improving get-github logins tests by specifying valueEncoding per put<commit_after>'use strict';
/*jshint asi: true */

var test      =  require('tap').test
  , xtend     =  require('xtend')
  , level     =  require('level-test')
  , sublevel  =  require('level-sublevel')
  , dump      =  require('level-dump')
  , getLogins =  require('../lib/get-github-logins')
  , sublevels = require('valuepack-core/mine/sublevels');

function toPut (o, valEnc) {
  return xtend(o, { type: 'put' }); 
}

function getPackagesBatch () {
  // each file contains subset of each user's packages
  return require('./fixtures/npm-packages-dominictarr.json')
    .concat(require('./fixtures/npm-packages-raynos.json'))
    .concat(require('./fixtures/npm-packages-substack.json'))
    .concat(require('./fixtures/npm-packages-thlorenz.json'))
    .concat(require('./fixtures/npm-packages-tj.json'))
    .map(toPut)
}

function setup (cb) {
  // contains metadata of these users
  var users = require('./fixtures/npm-users-tj-thlorenz-substack-dominictarr-raynos.json')

  // contains byOwner indexes for all batched packages
    , owners = require('./fixtures/byOwner-tj-thlorenz-substack-dominictarr-raynos.json')
    , usersBatch = users.map(toPut)
    , byOwnerBatch = owners.map(toPut).map(function (x) { return xtend(x, { valueEncoding: 'utf8' }) })
    , packagesBatch = getPackagesBatch()

  var db = sublevel(level({ mem: true })(null, { valueEncoding: 'json' }))

  var subnpm      =  sublevels(db).npm
    , npmPackages =  subnpm.packages
    , npmUsers    =  subnpm.users
    , npmByOwner  =  subnpm.byOwner


  npmPackages.batch(packagesBatch, function (err) {
    if (err) return cb(err);
    npmByOwner.batch(byOwnerBatch, function (err) {
      if (err) return cb(err);
      npmUsers.batch(usersBatch, function (err) {
        if (err) return cb(err);
        cb(null, db);
      })
    })
  })
}
{'fraction_non_alphanumeric': 0.15136097685067412, 'fraction_numerical': 0.0005087763927753752, 'mean_word_length': 3.2785636561479867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '891987', 'n_tokens_mistral': 1355, 'n_tokens_neox': 1288, 'n_words': 268}