<commit_before>var games = db.game4
var users = db.user2

var batchSize = 1000;
var i, t, timeStrings, times, it=0;
var dat = new Date().getTime() / 1000;
var max = users.count();

function hintWid(query) {
  return games.find(query).hint({wid:1}).length();
}

print("Denormalize counts")
users.find().sort({seenAt: -1}).forEach(function(user) {
  var uid = user['_id'];
  var count = {
    game: games.count({'uids': uid}),
    win: games.count({"wid": uid}),
    loss: games.count({"uids": uid, "s": { "$in": [ 30, 31, 35, 33 ] }, "wid": {"$ne": uid}}),
    draw: games.count({"uids": uid, "s": { "$in": [34, 32] }}),
    winH: hintWid({"wid": uid, "p.ai":{$exists:false}}),
    lossH: games.count({"uids": uid, "s": { "$in": [ 30, 31, 35, 33 ] }, "wid": {"$ne": uid}, "p.ai":{$exists:false}}),
    drawH: games.count({"uids": uid, "s": { "$in": [34, 32] }, "p.ai":{$exists:false}}),
    ai: games.count({"uids": uid, "p.ai":{$exists:true}}),
    rated: games.count({"uids": uid, "ra":true}),
  };
  users.update({"_id": uid}, {"$set":{count: count}});
  ++it;
  if (it % batchSize == 0) {
    var percent = Math.round((it / max) * 100);
    var dat2 = new Date().getTime() / 1000;
    var perSec = Math.round(batchSize / (dat2 - dat));
    dat = dat2;
    print((it / 1000) + "k " + percent + "% " + perSec + "/s");
  }
});<commit_msg>update user stats denormalization mongodb script<commit_after>var games = db.game5;
var users = db.user3;

var batchSize = 1000;
var i, t, timeStrings, times, it=0;
var dat = new Date().getTime() / 1000;
var max = users.count();

function hintWid(query) {
  return games.find(query).hint({wid:1}).length();
}

print("Denormalize counts");
users.find().sort({seenAt: -1}).forEach(function(user) {
  var uid = user._id;
  var count = {
    game: games.count({'us': uid}),
    win: games.count({"wid": uid}),
    loss: games.count({"us": uid, "s": { "$in": [ 30, 31, 35, 33 ] }, "wid": {"$ne": uid}}),
    draw: games.count({"us": uid, "s": { "$in": [34, 32] }}),
    winH: hintWid({"wid": uid, "p.ai":{$exists:false}}),
    lossH: games.count({"us": uid, "s": { "$in": [ 30, 31, 35, 33 ] }, "wid": {"$ne": uid}, "p.ai":{$exists:false}}),
    drawH: games.count({"us": uid, "s": { "$in": [34, 32] }, "p.ai":{$exists:false}}),
    ai: games.count({"us": uid, "p.ai":{$exists:true}}),
    rated: games.count({"us": uid, "ra":true}),
  };
  users.update({"_id": uid}, {"$set":{count: count}});
  ++it;
  if (it % batchSize === 0) {
    var percent = Math.round((it / max) * 100);
    var dat2 = new Date().getTime() / 1000;
    var perSec = Math.round(batchSize / (dat2 - dat));
    dat = dat2;
    print((it / 1000) + "k " + percent + "% " + perSec + "/s");
  }
});
{'fraction_non_alphanumeric': 0.26575240919199405, 'fraction_numerical': 0.0385470719051149, 'mean_word_length': 3.726795096322242, 'pattern_counts': {'":': 54, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2815842', 'n_tokens_mistral': 1138, 'n_tokens_neox': 1048, 'n_words': 257}