<commit_before>'use strict';
const gulp = require('gulp'),
  Promise = require('bluebird'),
  path = require('path'),
  mkdirp = require('mkdirp'),
  config = require('config'),
  levelDB = require('../lib/db/level'),
  _ = require('lodash'),
  fs = Promise.promisifyAll(require('fs')),
  log = require('../lib/logger'),
  CONFIG_PATH = path.resolve(__dirname, '..', 'config');

/**
 * Insert keys (app ids) and values (file path names) into leveldb.
 *
 * @datasetType The type of the dataset (e.g., listing, manifest, or ui).
 * @extension  file extension name (e.g., .xml, or .json).
 */
function insertToLevel(datasetType, extension) {
  const dirs = _.pluck(config.dataset[datasetType], 'target'),
    dirPaths = _.map(dirs, (dir) => {
      return path.resolve(CONFIG_PATH, dir);
    });
  return Promise.map(dirPaths, (dirPath) => {
    return fs.readdirAsync(dirPath)
        .map((fileName) => {
          const id = path.basename(fileName, extension),
            fileAbsPath = path.join(dirPath, fileName);
          return levelDB.dbGetAsync(id)
            .then((val) => {
              console.log(val);
              val[datasetType] = fileAbsPath;
              return levelDB.dbPutAsync(id, val);
            })
            .then(() => {
              log.info('Inserted %s path for %s', datasetType, id);
            })
            .catch((getErr) => {
              if (getErr.notFound) {
                const val = {};
                val[datasetType] = fileAbsPath;
                return levelDB.dbPutAsync(id, val)
                  .then(() => {
                    log.info('Inserted %s path for %s', datasetType,
                      id);
                  });
              }
              log.error('Failed to find %s ', id, getErr);
            });
        });
  })
    .then(() => {
      log.info('Finished inserting %s path values.', datasetType);
    })
    .catch((e) => {
      log.error('Error ', e);
      return Promise.reject(e);
    });
}

gulp.task('leveldb:create', (callback) => {
  // create leveldb data directory.
  mkdirp.sync(path.resolve(CONFIG_PATH, config.get(
    'dbConfig.leveldb.location')), { mode: '2775' });
  callback();
});

gulp.task('leveldb:readFullDB', () => {
  return levelDB.readFullDB();
});

gulp.task('leveldb:addListing', () => {
  return insertToLevel('listing', '.json');
});

gulp.task('leveldb:addManifest', () => {<commit_msg>refactor<commit_after>const gulp = require('gulp'),
  Promise = require('bluebird'),
  path = require('path'),
  mkdirp = require('mkdirp'),
  config = require('config'),
  levelDB = require('../lib/db/level'),
  _ = require('lodash'),
  fs = Promise.promisifyAll(require('fs')),
  log = require('../lib/logger'),
  CONFIG_PATH = path.resolve(__dirname, '..', 'config');

/**
 * Insert keys (app ids) and values (file path names) into leveldb.
 *
 * @datasetType The type of the dataset (e.g., listing, manifest, or ui).
 * @extension  file extension name (e.g., .xml, or .json).
 */
function insertToLevel(datasetType, extension) {
  const dirs = _.pluck(config.dataset[datasetType], 'target'),
    dirPaths = _.map(dirs, (dir) => {
      return path.resolve(CONFIG_PATH, dir);
    });
  return Promise.map(dirPaths, (dirPath) => {
    return fs.readdirAsync(dirPath)
        .map((fileName) => {
          const id = path.basename(fileName, extension),
            fileAbsPath = path.join(dirPath, fileName);
          return levelDB.dbGetAsync(id)
            .then((val) => {
              const result = val;
              log.info(result);
              result[datasetType] = fileAbsPath;
              return levelDB.dbPutAsync(id, result);
            })
            .then(() => {
              log.info('Inserted %s path for %s', datasetType, id);
            })
            .catch((getErr) => {
              if (getErr.notFound) {
                const val = {};
                val[datasetType] = fileAbsPath;
                return levelDB.dbPutAsync(id, val)
                  .then(() => {
                    log.info('Inserted %s path for %s', datasetType,
                      id);
                  });
              }
              log.error('Failed to find %s ', id, getErr);
              return Promise.reject(getErr);
            });
        });
  })
    .then(() => {
      log.info('Finished inserting %s path values.', datasetType);
    })
    .catch((e) => {
      log.error('Error ', e);
      return Promise.reject(e);
    });
}

gulp.task('leveldb:create', (callback) => {
  // create leveldb data directory.
  mkdirp.sync(path.resolve(CONFIG_PATH, config.get(
    'dbConfig.leveldb.location')), { mode: '2775' });
  callback();
});

gulp.task('leveldb:readFullDB', () => {
  return levelDB.readFullDB();
});

gulp.task('leveldb:addListing', () => {
  return insertToLevel('listing', '.json');
});

gulp.task('leveldb:addManifest', () => {
{'fraction_non_alphanumeric': 0.17206726825266613, 'fraction_numerical': 0.0016406890894175555, 'mean_word_length': 2.2340848806366047, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3775036', 'n_tokens_mistral': 1507, 'n_tokens_neox': 1484, 'n_words': 307}