<commit_before>var verbose_logging = (process.env.NODE_ENV !== "production");
var models  = require("../../models");
var express = require("express");
var router = express.Router();
var querystring = require("querystring");
var fs = require("fs");
var zlib = require("zlib");
var util = require("util");
var hash = require("../../misc/hash.js").hash;
var aws = require("../../misc/aws.js").aws();
var views = require("../../views/v1/models/_all.js").views;
var passport = require("passport");
passport.use(require("../../middleware/auth/passport-token.js").TokenStrategy);

router.route("/:guardian_id/checkins")
  .post(passport.authenticate("token",{session:false}), function(req,res) {

    var requestStartTime = (new Date()).valueOf();

    zlib.unzip(
      new Buffer(querystring.parse("gzipped="+req.body.meta).gzipped,"base64"),
      function(zLibError,zLibBuffer){
      if (!zLibError) {

        var json = JSON.parse(zLibBuffer.toString());
        if (verbose_logging) { console.log(json); }
      
        models.Guardian
          .findOne({ 
            where: { guid: req.params.guardian_id }
        }).then(function(dbGuardian){

          dbGuardian.last_check_in = new Date();
          dbGuardian.check_in_count = 1+dbGuardian.check_in_count;
          dbGuardian.save();

          var metaVersionArr = strArrToJSArr(json.software_version,"|","*"), versionJson = {};
          for (vInd in metaVersionArr) { versionJson[metaVersionArr[vInd][0]] = metaVersionArr[vInd][1]; }

          models.GuardianCheckIn
            .create({
              guardian_id: dbGuardian.id,
              software_versions: JSON.stringify(versionJson),
              measured_at: new Date(json.measured_at.replace(/ /g,"T")+json.timezone_offset),
              guardian_queued_checkins: parseInt(json.queued_checkins),
              guardian_skipped_checkins: parseInt(json.skipped_checkins),
              is_certified: dbGuardian.is_certified
          }).then(function(dbCheckIn){
            console.log("check-in: "+dbCheckIn.guid+" (guardian: "+dbGuardian.guid+") (version: "+"TBD"+")");

            // save guardian meta data

            // save guardian meta data transfer
            var metaDataTransfer = strArrToJSArr(json.data_transfer,"|","*");
            for (dtInd in metaDataTransfer) {
              models.GuardianMetaDataTransfer.create({
                  guardian_id: dbGuardian.id,
                  check_in_id: dbCheckIn.id,
                  started_at: new Date(metaDataTransfer[dtInd][0].replace(/ /g,"T")+json.timezone_offset),
                  ended_at: new Date(metaDataTransfer[dtInd][1].replace(/ /g,"T")+json.timezone_offset),
                  bytes_received: parseInt(metaDataTransfer[dtInd][2]),
                  bytes_sent: parseInt(metaDataTransfer[dtInd][3]),
                  total_bytes_received: parseInt(metaDataTransfer[dtInd][4]),<commit_msg>logging of input json sizes before and after decompression<commit_after>var verbose_logging = (process.env.NODE_ENV !== "production");
var models  = require("../../models");
var express = require("express");
var router = express.Router();
var querystring = require("querystring");
var fs = require("fs");
var zlib = require("zlib");
var util = require("util");
var hash = require("../../misc/hash.js").hash;
var aws = require("../../misc/aws.js").aws();
var views = require("../../views/v1/models/_all.js").views;
var passport = require("passport");
passport.use(require("../../middleware/auth/passport-token.js").TokenStrategy);

router.route("/:guardian_id/checkins")
  .post(passport.authenticate("token",{session:false}), function(req,res) {

    var requestStartTime = (new Date()).valueOf();

    zlib.unzip(
      new Buffer(querystring.parse("gzipped="+req.body.meta).gzipped,"base64"),
      function(zLibError,zLibBuffer){
      if (!zLibError) {

        var json = JSON.parse(zLibBuffer.toString());
        if (verbose_logging) { /*console.log(json);*/ }
          console.log("json length: "+req.body.meta.length+" "+json.length);
      
        models.Guardian
          .findOne({ 
            where: { guid: req.params.guardian_id }
        }).then(function(dbGuardian){

          dbGuardian.last_check_in = new Date();
          dbGuardian.check_in_count = 1+dbGuardian.check_in_count;
          dbGuardian.save();

          var metaVersionArr = strArrToJSArr(json.software_version,"|","*"), versionJson = {};
          for (vInd in metaVersionArr) { versionJson[metaVersionArr[vInd][0]] = metaVersionArr[vInd][1]; }

          models.GuardianCheckIn
            .create({
              guardian_id: dbGuardian.id,
              software_versions: JSON.stringify(versionJson),
              measured_at: new Date(json.measured_at.replace(/ /g,"T")+json.timezone_offset),
              guardian_queued_checkins: parseInt(json.queued_checkins),
              guardian_skipped_checkins: parseInt(json.skipped_checkins),
              is_certified: dbGuardian.is_certified
          }).then(function(dbCheckIn){
            console.log("check-in: "+dbCheckIn.guid+" (guardian: "+dbGuardian.guid+") (version: "+versionJson.api+")");

            // save guardian meta data

            // save guardian meta data transfer
            var metaDataTransfer = strArrToJSArr(json.data_transfer,"|","*");
            for (dtInd in metaDataTransfer) {
              models.GuardianMetaDataTransfer.create({
                  guardian_id: dbGuardian.id,
                  check_in_id: dbCheckIn.id,
                  started_at: new Date(metaDataTransfer[dtInd][0].replace(/ /g,"T")+json.timezone_offset),
                  ended_at: new Date(metaDataTransfer[dtInd][1].replace(/ /g,"T")+json.timezone_offset),
                  bytes_received: parseInt(metaDataTransfer[dtInd][2]),
                  bytes_sent: parseInt(metaDataTransfer[dtInd][3]),
                  total_bytes_received: parseInt(metaDataTransfer[dtInd][4]),
{'fraction_non_alphanumeric': 0.1514487870619946, 'fraction_numerical': 0.003706199460916442, 'mean_word_length': 2.913645352669743, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1340825', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1737, 'n_words': 285}