<commit_before>var CouchClient = require('../lib/couch-client');
var Data = require('../data');
var _ = require('underscore');
var async = require('async');


// Apply Filter Stack
// --------

var applyFilters = function(filters, nodes, mode, ctx, callback) {
  var that = this;
  var filteredNodes = {};
  if (!filters || filters.length === 0) return callback(null, nodes); // Skip
  
  async.forEach(_.keys(nodes), function(key, callback) {
    var idx = 0;
    function callLayer(node, callback) {
      if (!node) return callback(null); // skip rejected nodes
      if (filters[idx]) {
        filters[idx][mode].call(that, node, function(n) {
          idx += 1;
          callLayer(n, function(n) {
            callback(n);
          });
        }, ctx);
      } else {
        callback(node);
      }
    }
    callLayer(nodes[key], function(node) {
      if (node) filteredNodes[node._id] = node;
      callback();
    });
  }, function(err) {
    err ? callback(err) : callback(null, filteredNodes);
  });
};


var CouchAdapter = function(graph, config, callback) {
  var db = CouchClient(config.url);
  var self = {};
  
  // Setup index views for type nodes
  // --------
  
  function setupType(node, callback) {
    
    // Extract local typename
    var typename = node._id.split('/')[2];
    views = {
      "all": {
        "properties": ["_id"],
        "map": "function(doc) { if (doc.type.indexOf('"+node._id+"')>=0) { emit([doc._id], doc); } }"
      }
    };
    
    // Setup validation function
    var validatorFn = "function(newDoc, oldDoc, userCtx) {\n";
    validatorFn += "if (newDoc.type.indexOf('"+node._id+"')>=0) {\n";
    _.each(node.properties, function(property, key) {
      if (property.required) validatorFn += "if (!newDoc['"+key+"']) throw({forbidden : '"+key+" is missing'});\n";
      if (property.validator) validatorFn += "if (!new RegExp("+JSON.stringify(property.validator)+").test(newDoc."+key+")) throw({forbidden: '"+key+" is invalid'});"
    });
    validatorFn += "}}";
    
    if (node.indexes) {
      _.each(node.indexes, function(properties, indexName) {
        var keyExpr = "["+ _.map(properties, function(p) { return "doc."+p;}).join(',')+ "]";
        views[indexName] = {
          "properties": properties,
          "map": "function(doc) { if (doc.type.indexOf('"+node._id+"')>=0) { emit("+keyExpr+", doc); } }"
        };
      });
    }
    
    db.save({
      _id: '_design/'+typename,
      views: views,
      validate_doc_update: validatorFn
    }, {force: true}, function (err, doc) {
      err ? callback(err) : callback();
    });
  };
  
  // flush
  // --------------

  // Flush the database
  
  self.flush = function(callback) {
    db.request("DELETE", db.uri.pathname, function (err) {
      db.request("PUT", db.uri.pathname, function(err) {
        err ? callback(err) 
            : db.save({
                _id: '_design/type',
                views: {
                  "all": {
                    "map": "function(doc) { if (doc.type === \"/type/type\") emit(doc._id, doc); }"<commit_msg>Consider deleted documents in validation functions.<commit_after>var CouchClient = require('../lib/couch-client');
var Data = require('../data');
var _ = require('underscore');
var async = require('async');


// Apply Filter Stack
// --------

var applyFilters = function(filters, nodes, mode, ctx, callback) {
  var that = this;
  var filteredNodes = {};
  if (!filters || filters.length === 0) return callback(null, nodes); // Skip
  
  async.forEach(_.keys(nodes), function(key, callback) {
    var idx = 0;
    function callLayer(node, callback) {
      if (!node) return callback(null); // skip rejected nodes
      if (filters[idx]) {
        filters[idx][mode].call(that, node, function(n) {
          idx += 1;
          callLayer(n, function(n) {
            callback(n);
          });
        }, ctx);
      } else {
        callback(node);
      }
    }
    callLayer(nodes[key], function(node) {
      if (node) filteredNodes[node._id] = node;
      callback();
    });
  }, function(err) {
    err ? callback(err) : callback(null, filteredNodes);
  });
};


var CouchAdapter = function(graph, config, callback) {
  var db = CouchClient(config.url);
  var self = {};
  
  // Setup index views for type nodes
  // --------
  
  function setupType(node, callback) {
    
    // Extract local typename
    var typename = node._id.split('/')[2];
    views = {
      "all": {
        "properties": ["_id"],
        "map": "function(doc) { if (doc.type.indexOf('"+node._id+"')>=0) { emit([doc._id], doc); } }"
      }
    };
    
    // Setup validation function
    var validatorFn = "function(newDoc, oldDoc, userCtx) {\n";
    validatorFn += "if (newDoc.type && newDoc.type.indexOf('"+node._id+"')>=0) {\n";
    _.each(node.properties, function(property, key) {
      if (property.required) validatorFn += "if (!newDoc['"+key+"']) throw({forbidden : '"+key+" is missing'});\n";
      if (property.validator) validatorFn += "if (!new RegExp("+JSON.stringify(property.validator)+").test(newDoc."+key+")) throw({forbidden: '"+key+" is invalid'});"
    });
    validatorFn += "}}";
    
    if (node.indexes) {
      _.each(node.indexes, function(properties, indexName) {
        var keyExpr = "["+ _.map(properties, function(p) { return "doc."+p;}).join(',')+ "]";
        views[indexName] = {
          "properties": properties,
          "map": "function(doc) { if (doc.type && doc.type.indexOf('"+node._id+"')>=0) { emit("+keyExpr+", doc); } }"
        };
      });
    }
    
    db.save({
      _id: '_design/'+typename,
      views: views,
      validate_doc_update: validatorFn
    }, {force: true}, function (err, doc) {
      err ? callback(err) : callback();
    });
  };
  
  // flush
  // --------------

  // Flush the database
  
  self.flush = function(callback) {
    db.request("DELETE", db.uri.pathname, function (err) {
      db.request("PUT", db.uri.pathname, function(err) {
        err ? callback(err) 
            : db.save({
                _id: '_design/type',
                views: {
                  "all": {
                    "map": "function(doc) { if (doc.type === \"/type/type\") emit(doc._id, doc); }"
{'fraction_non_alphanumeric': 0.1975566629159299, 'fraction_numerical': 0.002250442051117184, 'mean_word_length': 2.413055403181569, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '324736', 'n_tokens_mistral': 2000, 'n_tokens_neox': 1974, 'n_words': 423}