<commit_before>// Import forum votes
//

'use strict';

const Promise   = require('bluebird');
const co        = require('co');
const mongoose  = require('mongoose');
const progress  = require('./utils').progress;
const POST      = 1; // content type for posts


module.exports = co.wrap(function* (N) {
  //
  // Establish MySQL connection
  //
  let conn = yield N.vbconvert.getConnection();

  //
  // Fetch all users
  //
  let users = {};

  (yield N.models.users.User.find().lean(true)).forEach(user => {
    users[user.hid] = user;
  });


  let rows = yield conn.query('SELECT count(*) AS count FROM votes');

  let bar = progress(' votes :current/:total [:bar] :percent', rows[0].count);

  let userids = yield conn.query(`
    SELECT fromuserid FROM votes
    GROUP BY fromuserid
    ORDER BY fromuserid ASC
  `);

  yield Promise.map(userids, co.wrap(function* (row) {
    // ignore votes casted by deleted users
    if (!users[row.fromuserid]) return;

    let rows = yield conn.query(`
      SELECT targetid,vote,fromuserid,touserid,date
      FROM votes
      WHERE fromuserid = ? AND contenttypeid = ?
    `, [ row.fromuserid, POST ]);

    let bulk = N.models.users.Vote.collection.initializeUnorderedBulkOp();
    let count = 0;

    for (var i = 0; i < rows; i++) {
      bar.tick();

      // ignore votes casted for deleted users
      if (!users[row.touserid]) continue;

      let post_mapping = yield N.models.vbconvert.PostMapping.findOne({
        mysql_id: row.targetid
      }).lean(true);

      count++;
      bulk.find({
        from:  users[row.fromuserid]._id,
        to:    users[row.touserid]._id,
        'for': post_mapping.post_id,
        type:  N.models.users.Vote.types.FORUM_POST
      }).upsert().update({
        $setOnInsert: {
          _id:   new mongoose.Types.ObjectId(row.date),
          from:  users[row.fromuserid]._id,
          to:    users[row.touserid]._id,
          'for': post_mapping.post_id,
          type:  N.models.users.Vote.types.FORUM_POST,
          hb:    users[row.fromuserid].hb,
          value: Number(row.vote)
        }
      });
    }

    if (!count) return;

    yield new Promise((resolve, reject) => {
      bulk.execute(err => err ? reject(err) : resolve());
    });
  }), { concurrency: 100 });

  bar.terminate();
  conn.release();
  N.logger.info('Vote import finished');
});
<commit_msg>Fix vote import<commit_after>// Import forum votes
//

'use strict';

const Promise   = require('bluebird');
const co        = require('co');
const mongoose  = require('mongoose');
const progress  = require('./utils').progress;
const POST      = 1; // content type for posts


module.exports = co.wrap(function* (N) {
  //
  // Establish MySQL connection
  //
  let conn = yield N.vbconvert.getConnection();

  //
  // Fetch all users
  //
  let users = {};

  (yield N.models.users.User.find().lean(true)).forEach(user => {
    users[user.hid] = user;
  });


  let rows = yield conn.query('SELECT count(*) AS count FROM votes');

  let bar = progress(' votes :current/:total [:bar] :percent', rows[0].count);

  let userids = yield conn.query(`
    SELECT fromuserid FROM votes
    GROUP BY fromuserid
    ORDER BY fromuserid ASC
  `);

  yield Promise.map(userids, co.wrap(function* (userid_row) {
    let fromuserid = userid_row.fromuserid;

    // ignore votes casted by deleted users
    if (!users[fromuserid]) return;

    let rows = yield conn.query(`
      SELECT targetid,vote,fromuserid,touserid,date
      FROM votes
      WHERE fromuserid = ? AND contenttypeid = ?
    `, [ fromuserid, POST ]);

    let bulk = N.models.users.Vote.collection.initializeUnorderedBulkOp();
    let count = 0;

    for (let i = 0; i < rows.length; i++) {
      let row = rows[i];

      bar.tick();

      // ignore votes casted for deleted users
      if (!users[row.touserid]) continue;

      let post_mapping = yield N.models.vbconvert.PostMapping.findOne({
        mysql_id: row.targetid
      }).lean(true);

      // voted for non-existent or not imported post
      if (!post_mapping) continue;

      count++;
      bulk.find({
        from:  users[row.fromuserid]._id,
        to:    users[row.touserid]._id,
        'for': post_mapping.post_id,
        type:  N.models.users.Vote.types.FORUM_POST
      }).upsert().update({
        $setOnInsert: {
          _id:   new mongoose.Types.ObjectId(row.date),
          from:  users[row.fromuserid]._id,
          to:    users[row.touserid]._id,
          'for': post_mapping.post_id,
          type:  N.models.users.Vote.types.FORUM_POST,
          hb:    users[row.fromuserid].hb,
          value: Number(row.vote)
        }
      });
    }

    if (!count) return;

    yield new Promise((resolve, reject) => {
      bulk.execute(err => err ? reject(err) : resolve());
    });
  }), { concurrency: 100 });

  bar.terminate();
  conn.release();
  N.logger.info('Vote import finished');
});

{'fraction_non_alphanumeric': 0.14631664631664631, 'fraction_numerical': 0.002849002849002849, 'mean_word_length': 2.4858156028368796, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7511292', 'n_tokens_mistral': 1653, 'n_tokens_neox': 1594, 'n_words': 374}