<commit_before>// Export old to new forum id mappings into levelDB
//

'use strict';

const _           = require('lodash');
const co          = require('co');
const level       = require('level');
const mkdirp      = require('mkdirp');
const path        = require('path');
const pump        = require('pump');
const through2    = require('through2');
const progress    = require('./_lib/utils').progress;


////////////////////////////////////////////////////////////////////////////////


module.exports.parserParameters  = {
  addHelp:      true,
  help:         'export mappings into levelDB',
  description:  'Export old to new forum id mappings into levelDB'
};


module.exports.commandLineArguments = [
  {
    args: [ '-d', '--dest' ],
    options: {
      help:     'Database path',
      type:     'string',
      required: true
    }
  }
];


module.exports.run = co.wrap(function* (N, args) {
  let total, bar, ldb = {};

  yield N.wire.emit('init:models', N);

  mkdirp.sync(args.dest);

  //
  // Export topic mappings
  //
  N.logger.info('Loading topics into memory');

  [ 'topics', 'posts', 'avatars', 'albums', 'attachments', 'filedataids', 'pictureaids', 'blogaids' ].forEach(name => {
    ldb[name] = level(path.join(args.dest, name), {
      keyEncoding:   'json',
      valueEncoding: 'json'
    });
  });

  let all_topics    = yield N.models.forum.Topic.find().select('section hid').lean(true);
  let topics_by_hid = _.keyBy(all_topics, 'hid');
  let topics_by_id  = _.keyBy(all_topics, '_id');

  let sections_by_id = _.keyBy(
    yield N.models.forum.Section.find().select('_id hid').lean(true),
    '_id'
  );

  let batch = ldb.topics.batch();

  Object.keys(topics_by_hid).forEach(topic_hid => {
    let section_id = topics_by_hid[topic_hid].section;
<commit_msg>Don't use json for leveldb keys<commit_after>// Export old to new forum id mappings into levelDB
//

'use strict';

const _           = require('lodash');
const co          = require('co');
const level       = require('level');
const mkdirp      = require('mkdirp');
const path        = require('path');
const pump        = require('pump');
const through2    = require('through2');
const progress    = require('./_lib/utils').progress;


////////////////////////////////////////////////////////////////////////////////


module.exports.parserParameters  = {
  addHelp:      true,
  help:         'export mappings into levelDB',
  description:  'Export old to new forum id mappings into levelDB'
};


module.exports.commandLineArguments = [
  {
    args: [ '-d', '--dest' ],
    options: {
      help:     'Database path',
      type:     'string',
      required: true
    }
  }
];


module.exports.run = co.wrap(function* (N, args) {
  let total, bar, ldb = {};

  yield N.wire.emit('init:models', N);

  mkdirp.sync(args.dest);

  //
  // Export topic mappings
  //
  N.logger.info('Loading topics into memory');

  [ 'topics', 'posts', 'avatars', 'albums', 'attachments', 'filedataids', 'pictureaids', 'blogaids' ].forEach(name => {
    ldb[name] = level(path.join(args.dest, name), {
      valueEncoding: 'json'
    });
  });

  let all_topics    = yield N.models.forum.Topic.find().select('section hid').lean(true);
  let topics_by_hid = _.keyBy(all_topics, 'hid');
  let topics_by_id  = _.keyBy(all_topics, '_id');

  let sections_by_id = _.keyBy(
    yield N.models.forum.Section.find().select('_id hid').lean(true),
    '_id'
  );

  let batch = ldb.topics.batch();

  Object.keys(topics_by_hid).forEach(topic_hid => {
    let section_id = topics_by_hid[topic_hid].section;

{'fraction_non_alphanumeric': 0.2, 'fraction_numerical': 0.0011188811188811189, 'mean_word_length': 2.5405940594059406, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2245389', 'n_tokens_mistral': 1182, 'n_tokens_neox': 1148, 'n_words': 257}