<commit_before>var async = require("async");
var Promise = require("promise");

var BaseBackend = function() {
}

BaseBackend.prototype = {
  isReference: function(field, value) {
    if (field === 'origin' || field === 'foreign_id') {
      return false;
    }
    return (typeof(value) === 'string' && value.split(':').length == 2);
  },
  iterActivityFields: function(activities, filter, fn) {
    for (var i in activities) {
      var activity = activities[i];
      for (var field in activity) {
        if(!filter(field, activity[field])) continue;
        var args = {
          'activity': activity,
          'field': field
        };
        fn(args);
      }
    }
  },
  iterActivityFieldsWithObjects: function(activities, fn) {
    this.iterActivityFields(activities, function(field, value){ return (value !==  null && typeof(value) === 'object');}, fn);
  },
  iterActivityFieldsWithReferences: function(activities, fn) {
    var self = this;
    this.iterActivityFields(activities, this.isReference, function(args) {
      var field = args['field'];
      args['modelRef'] = args['activity'][field].split(":")[0];
      args['instanceRef'] = args['activity'][field].split(":")[1];
      fn(args);
    });
  },
  collectReferences: function(activities) {
    var modelReferences = {};
    this.iterActivityFieldsWithReferences(activities, function(args) {
      if (modelReferences[args.modelRef]){
        modelReferences[args.modelRef].push(args.instanceRef);
      } else {
        modelReferences[args.modelRef] = [args.instanceRef];
      }
    });
    return modelReferences;
  },
  retreiveObjects: function (references, callback) {
    var objects = {};
    var self = this;
    async.each(Object.keys(references),
      function(modelRef, done){
        var refs = references[modelRef];
        var modelClass = self.getClassFromRef(modelRef);
        if (typeof(modelClass) === 'undefined') return done();
        if (typeof(objects[modelRef]) === 'undefined') objects[modelRef] = {};
        self.loadFromStorage(modelClass, refs, function(err, objectsIds) {
          for(var k in objectsIds){
            objects[modelRef][k] = objectsIds[k];
          }
          done(err);
        });
      },
      function(err){
        callback(err, objects);
      }
    );
  },
  enrichActivities: function(activities) {
    // Return a Promise instead of using node style callbacks (_enrichActivities accepts one argument + callback)
    return Promise.denodeify(this._enrichActivities, 1).call(this, activities);
  },
  _enrichActivities: function(activities, callback) {
    var self = this;
    var references = this.collectReferences(activities);
    this.retreiveObjects(references, function(err, objects) {
      self.iterActivityFieldsWithReferences(activities, function(args) {
        if (objects[args.modelRef] && objects[args.modelRef][args.instanceRef] && args.field !== 'foreign_id'){
          args.activity[args.field] = objects[args.modelRef][args.instanceRef];
        }
      });
      callback(err, activities);
    });
  },
  enrichAggregatedActivities: function(aggregatedActivities) {
    // Return a Promise instead of using node style callbacks (_enrichAggregatedActivities accepts one argument + callback)
    return Promise.denodeify(this._enrichAggregatedActivities, 1).call(this, aggregatedActivities);
  },
  _enrichAggregatedActivities: function(aggregatedActivities, callback) {
    var references = {};
    var enrichments = [];
    var self = this;
    for (var i in aggregatedActivities) {
      enrichments.push(
        (function(aggregated){
          return function(done) {
            self.enrichActivities(aggregated['activities'])
              .then(done.bind(this, null), done);
          }
        })(aggregatedActivities[i]))
    }
    async.parallel(enrichments, function(err) { callback(err, aggregatedActivities); });
  },
  serializeActivities: function(activities){
    var self = this;
    this.iterActivityFieldsWithObjects(activities, function(args) {
      var value = args.activity[args.field];
      args.activity[args.field] = self.serializeValue(value);
    });
  },
  // Backend specific functions
  loadFromStorage: function(objectsIds, callback) {},
  serializeValue: function(value) {<commit_msg>added serialize activity<commit_after>var async = require("async");
var Promise = require("promise");

var BaseBackend = function() {
}

BaseBackend.prototype = {
  isReference: function(field, value) {
    if (field === 'origin' || field === 'foreign_id') {
      return false;
    }
    return (typeof(value) === 'string' && value.split(':').length == 2);
  },
  iterActivityFields: function(activities, filter, fn) {
    for (var i in activities) {
      var activity = activities[i];
      for (var field in activity) {
        if(!filter(field, activity[field])) continue;
        var args = {
          'activity': activity,
          'field': field
        };
        fn(args);
      }
    }
  },
  iterActivityFieldsWithObjects: function(activities, fn) {
    this.iterActivityFields(activities, function(field, value){ return (value !==  null && typeof(value) === 'object');}, fn);
  },
  iterActivityFieldsWithReferences: function(activities, fn) {
    var self = this;
    this.iterActivityFields(activities, this.isReference, function(args) {
      var field = args['field'];
      args['modelRef'] = args['activity'][field].split(":")[0];
      args['instanceRef'] = args['activity'][field].split(":")[1];
      fn(args);
    });
  },
  collectReferences: function(activities) {
    var modelReferences = {};
    this.iterActivityFieldsWithReferences(activities, function(args) {
      if (modelReferences[args.modelRef]){
        modelReferences[args.modelRef].push(args.instanceRef);
      } else {
        modelReferences[args.modelRef] = [args.instanceRef];
      }
    });
    return modelReferences;
  },
  retreiveObjects: function (references, callback) {
    var objects = {};
    var self = this;
    async.each(Object.keys(references),
      function(modelRef, done){
        var refs = references[modelRef];
        var modelClass = self.getClassFromRef(modelRef);
        if (typeof(modelClass) === 'undefined') return done();
        if (typeof(objects[modelRef]) === 'undefined') objects[modelRef] = {};
        self.loadFromStorage(modelClass, refs, function(err, objectsIds) {
          for(var k in objectsIds){
            objects[modelRef][k] = objectsIds[k];
          }
          done(err);
        });
      },
      function(err){
        callback(err, objects);
      }
    );
  },
  enrichActivities: function(activities) {
    // Return a Promise instead of using node style callbacks (_enrichActivities accepts one argument + callback)
    return Promise.denodeify(this._enrichActivities, 1).call(this, activities);
  },
  _enrichActivities: function(activities, callback) {
    var self = this;
    var references = this.collectReferences(activities);
    this.retreiveObjects(references, function(err, objects) {
      self.iterActivityFieldsWithReferences(activities, function(args) {
        if (objects[args.modelRef] && objects[args.modelRef][args.instanceRef] && args.field !== 'foreign_id'){
          args.activity[args.field] = objects[args.modelRef][args.instanceRef];
        }
      });
      callback(err, activities);
    });
  },
  enrichAggregatedActivities: function(aggregatedActivities) {
    // Return a Promise instead of using node style callbacks (_enrichAggregatedActivities accepts one argument + callback)
    return Promise.denodeify(this._enrichAggregatedActivities, 1).call(this, aggregatedActivities);
  },
  _enrichAggregatedActivities: function(aggregatedActivities, callback) {
    var references = {};
    var enrichments = [];
    var self = this;
    for (var i in aggregatedActivities) {
      enrichments.push(
        (function(aggregated){
          return function(done) {
            self.enrichActivities(aggregated['activities'])
              .then(done.bind(this, null), done);
          }
        })(aggregatedActivities[i]))
    }
    async.parallel(enrichments, function(err) { callback(err, aggregatedActivities); });
  },
  serializeActivities: function(activities){
    var self = this;
    this.iterActivityFieldsWithObjects(activities, function(args) {
      var value = args.activity[args.field];
      args.activity[args.field] = self.serializeValue(value);
    });
  },
  serializeActivity: function(activity) {
      this.serializeActivities([activity]);
  },
  // Backend specific functions
  loadFromStorage: function(objectsIds, callback) {},
  serializeValue: function(value) {
{'fraction_non_alphanumeric': 0.13602771362586605, 'fraction_numerical': 0.0011547344110854503, 'mean_word_length': 3.3808801213960544, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5732208', 'n_tokens_mistral': 2498, 'n_tokens_neox': 2460, 'n_words': 499}