<commit_before>/**
 * Copyright 2013-2017  <NAME>
 *
 * This file is part of GenieACS.
 *
 * GenieACS is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * GenieACS is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
 */
"use strict";

const vm = require("vm");
const crypto = require("crypto");

const later = require("later");

const db = require("./db");
const cache = require("./cache");
const query = require("./query");
const logger = require("./logger");

const REFRESH = 3000;

let nextRefresh = 1;
let hash = null;
let presets, provisions, virtualParameters, files;


function computeHash() {
  // MD5 hash for presets, provisions, virtual parameters for detecting changes
  let h = crypto.createHash("md5");
  for (let p of presets) {
    h.update(JSON.stringify(p.name));
    h.update(JSON.stringify(p.channel));
    h.update(JSON.stringify(p.schedule));
    h.update(JSON.stringify(p.events));
    h.update(JSON.stringify(p.precondition));
    h.update(JSON.stringify(p.provisions));
  }

  let keys;

  keys = Object.keys(provisions).sort();
  h.update(JSON.stringify(keys));
  for (let k of keys)
    h.update(provisions[k].md5);

  keys = Object.keys(virtualParameters).sort();
  h.update(JSON.stringify(keys));
  for (let k of keys)
    h.update(virtualParameters[k].md5);

  hash = h.digest("hex");
}


function refresh(callback) {
  if (!nextRefresh)
    return setTimeout(function() {refresh(callback);}, 20);

  nextRefresh = 0;
  const now = Date.now();

  cache.get("presets_hash", function(err, res) {
    if (err)
      return callback(err);

    if (hash && res === hash) {
      nextRefresh = now + (REFRESH - (now % REFRESH));
      return callback();
    }

    cache.lock("presets_hash_lock", 3, function(err, unlockOrExtend) {
      if (err)
        return callback(err);

      let counter = 3;

      counter += 2;
      db.getPresets(function(err, res) {
        if (err) {
          if (counter & 1)
            callback(err);
          return counter = 0;
        }

        db.getObjects(function(err, objects) {
          if (err) {
            if (counter & 1)
              callback(err);
            return counter = 0;
          }

          res.sort((a, b) => {
            if (a.weight === b.weight)
              return a._id > b._id;
            else
              return a.weight - b.weight;
          });

          presets = [];
          for (let preset of res) {
            let schedule = null;
            if (preset.schedule) {
              let parts = preset.schedule.trim().split(/\s+/);
              schedule = {
                md5: crypto.createHash("md5").update(preset.schedule).digest("hex")
              };
<commit_msg>Consider all object params as keys if none exists<commit_after>/**
 * Copyright 2013-2017  <NAME>
 *
 * This file is part of GenieACS.
 *
 * GenieACS is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * GenieACS is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
 */
"use strict";

const vm = require("vm");
const crypto = require("crypto");

const later = require("later");

const db = require("./db");
const cache = require("./cache");
const query = require("./query");
const logger = require("./logger");

const REFRESH = 3000;

let nextRefresh = 1;
let hash = null;
let presets, provisions, virtualParameters, files;


function computeHash() {
  // MD5 hash for presets, provisions, virtual parameters for detecting changes
  let h = crypto.createHash("md5");
  for (let p of presets) {
    h.update(JSON.stringify(p.name));
    h.update(JSON.stringify(p.channel));
    h.update(JSON.stringify(p.schedule));
    h.update(JSON.stringify(p.events));
    h.update(JSON.stringify(p.precondition));
    h.update(JSON.stringify(p.provisions));
  }

  let keys;

  keys = Object.keys(provisions).sort();
  h.update(JSON.stringify(keys));
  for (let k of keys)
    h.update(provisions[k].md5);

  keys = Object.keys(virtualParameters).sort();
  h.update(JSON.stringify(keys));
  for (let k of keys)
    h.update(virtualParameters[k].md5);

  hash = h.digest("hex");
}


function refresh(callback) {
  if (!nextRefresh)
    return setTimeout(function() {refresh(callback);}, 20);

  nextRefresh = 0;
  const now = Date.now();

  cache.get("presets_hash", function(err, res) {
    if (err)
      return callback(err);

    if (hash && res === hash) {
      nextRefresh = now + (REFRESH - (now % REFRESH));
      return callback();
    }

    cache.lock("presets_hash_lock", 3, function(err, unlockOrExtend) {
      if (err)
        return callback(err);

      let counter = 3;

      counter += 2;
      db.getPresets(function(err, res) {
        if (err) {
          if (counter & 1)
            callback(err);
          return counter = 0;
        }

        db.getObjects(function(err, objects) {
          if (err) {
            if (counter & 1)
              callback(err);
            return counter = 0;
          }

          // If no keys are defined, consider all parameters as keys to keep the
          // same behavior from v1.0
          for (let obj of objects) {
            if (!obj["_keys"] || !obj["_keys"].length)
              obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
          }

          res.sort((a, b) => {
            if (a.weight === b.weight)
              return a._id > b._id;
            else
              return a.weight - b.weight;
          });

          presets = [];
          for (let preset of res) {
            let schedule = null;
            if (preset.schedule) {
              let parts = preset.schedule.trim().split(/\s+/);
              schedule = {
                md5: crypto.createHash("md5").update(preset.schedule).digest("hex")
              };

{'fraction_non_alphanumeric': 0.13176574977817213, 'fraction_numerical': 0.00916888494528246, 'mean_word_length': 2.478909465020576, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1412050', 'n_tokens_mistral': 2085, 'n_tokens_neox': 1977, 'n_words': 614}