<commit_before>var util = require('util');
var _ = require('underscore');
var async = require('async');
var syncUtil = require('./util');
var metrics = require('./sync-metrics');
var datasets = require('./datasets');

var syncStorage, dataHandlers, metricsClient, hashProvider;

function listDatafromBackend(datasetId, queryParams, metaData, cb) {
  dataHandlers.doList(datasetId, queryParams, metaData, function(err, records){
    if (err) {
      return cb(err);
    } else {
      //now we have the results from the backend, we should:
      //iterate through them
      //get the hash of each record, try save them in redis
      //generate the global hash
      //save the info in redis
      var recordsWithHash = _.map(records, function(data, uid){
        var recHash = hashProvider.recordHash(datasetId, data);
        return {
          uid: uid,
          data: data,
          hash: recHash
        };
      });
      return cb(null, recordsWithHash);
    }
  });
}

function listDataWithTimeout(params, timeout, cb) {
  async.race([
    function(timeoutCallback) {
      setTimeout(function(){
        timeoutCallback(new Error('Timeout error occured when try to list data from backend for datasetClient ' + params.datasetClientId));
      }, timeout);
    },
    function(listCallback) {
      listDatafromBackend(params.datasetId, params.queryParams, params.metaData, listCallback);
    }
  ], cb);
}

function recordProcessTime(startTime, success) {
  var endTime = Date.now();
  var totalTime = endTime - startTime;
  metricsClient.gauge(metrics.KEYS.SYNC_REQUEST_TOTAL_PROCESS_TIME, {success: success}, totalTime);
}

function markDatasetClientAsCompleted(datasetClientId, startTime, callback) {
  syncStorage.updateDatasetClient(datasetClientId, {syncLoopEnd: Date.now(), syncCompleted: true, syncScheduled: null}, function(err){
    if (err) {
      // TODO: datasetId here isn't defined, is it?
      syncUtil.doLog(datasetId, "debug", "Error when update dataset client with id " + datasetClientId + ". error = " + util.inspect(err));
    }<commit_msg>🐛 Fix a doLog param that was not in scope<commit_after>var util = require('util');
var _ = require('underscore');
var async = require('async');
var syncUtil = require('./util');
var metrics = require('./sync-metrics');
var datasets = require('./datasets');

var syncStorage, dataHandlers, metricsClient, hashProvider;

function listDatafromBackend(datasetId, queryParams, metaData, cb) {
  dataHandlers.doList(datasetId, queryParams, metaData, function(err, records){
    if (err) {
      return cb(err);
    } else {
      //now we have the results from the backend, we should:
      //iterate through them
      //get the hash of each record, try save them in redis
      //generate the global hash
      //save the info in redis
      var recordsWithHash = _.map(records, function(data, uid){
        var recHash = hashProvider.recordHash(datasetId, data);
        return {
          uid: uid,
          data: data,
          hash: recHash
        };
      });
      return cb(null, recordsWithHash);
    }
  });
}

function listDataWithTimeout(params, timeout, cb) {
  async.race([
    function(timeoutCallback) {
      setTimeout(function(){
        timeoutCallback(new Error('Timeout error occured when try to list data from backend for datasetClient ' + params.datasetClientId));
      }, timeout);
    },
    function(listCallback) {
      listDatafromBackend(params.datasetId, params.queryParams, params.metaData, listCallback);
    }
  ], cb);
}

function recordProcessTime(startTime, success) {
  var endTime = Date.now();
  var totalTime = endTime - startTime;
  metricsClient.gauge(metrics.KEYS.SYNC_REQUEST_TOTAL_PROCESS_TIME, {success: success}, totalTime);
}

function markDatasetClientAsCompleted(datasetClientId, startTime, callback) {
  syncStorage.updateDatasetClient(datasetClientId, {syncLoopEnd: Date.now(), syncCompleted: true, syncScheduled: null}, function(err){
    if (err) {
      syncUtil.doLog(syncUtil.SYNC_LOGGER, "debug", "Error when update dataset client with id " + datasetClientId + ". error = " + util.inspect(err));
    }
{'fraction_non_alphanumeric': 0.11710621824340672, 'fraction_numerical': 0.0, 'mean_word_length': 3.6397306397306397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1974147', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1175, 'n_words': 330}