<commit_before>var MongoClient = require('mongodb').MongoClient;
var projector = require("nodetiles-core").projector;
var __ = require("lodash");


var MongoSource = function(options) {
  this._projection = projector.util.cleanProjString(options.projection || 'EPSG:4326');
  this._connectionString = options.connectionString; // required
  this._collectionName = options.collectionName;     // required
  this._geoKey = options.key;                        // required - name of the geodata field
  this._query =   options.query;                     // required - a query to get started with
  this._select = options.select;                     // required - fields to select
  this._filter = options.filter;                     // optional - key to filter data
  this.sourceName = options.name || 'localdata';
  return this;
};

MongoSource.prototype = {
  constructor: MongoSource,

  getShapes: function(minX, minY, maxX, maxY, mapProjection, callback) {

    // First, we need to turn the coordinates into a bounds query for Mongo
    var min = [minX, minY];
    var max = [maxX, maxY];

    // project request coordinates into data coordinates
    if (mapProjection !== this._projection) {
      min = projector.project.Point(mapProjection, this._projection, min);
      max = projector.project.Point(mapProjection, this._projection, max);
    }

    var query = this._query || {};
    var parsedBbox = [[min[0], min[1]], [max[0],  max[1]]];
    query[this._geoKey] = { '$within': { '$box': parsedBbox } };

    // TODO: set autoreconnect
    MongoClient.connect(this._connectionString, {
      server: {
        auto_reconnect: true
      }
    }, function(err, db) {
      if(err) {
        console.log("Mongo error:", err);
      }

      start = Date.now();
      var stream = db.collection(this._collectionName)
        .find(this._query, this._select)
        .stream();

      var cursor = db.collection(this._collectionName)
        .find(this._query, this._select);

      console.log("Query %j %j", this._query, this._select);

      var features = [];

      var self = this;

      /**
       * Turn stored a stored parcel result into geoJSON
       * TODO: Can save time and memory by not creating a new object here.
       * @param  {Object} item  A single response
       * @return {Object}       A single response structured as geoJSON
       */
      function resultToGeoJSON(item, filter) {
        item.type = 'Feature';

        // Get the shape
        // Or if there isn't one, use the centroid.
        if (item.geo_info.geometry !== undefined) {
          item.id = item.parcel_id;
          item.geometry = item.geo_info.geometry;
        }else {
          item.id = item._id;
          item.geometry = {};
          item.geometry.type = 'Point';
          item.geometry.coordinates = item.geo_info.centroid;
        }

        // item.properties = item;
        item.properties = {
          geometry: item.geo_info.geometry,
          name: item.geo_info.humanReadableName
        };

        // Clean up a bit
        delete item.geo_info.centroid;
        delete item.geo_info.geometry;
        delete item.responses;

        // If there is a filer, we also want the key easily accessible.
        if(filter) {
          // TODO: Handle the undefined condition
          if(item.hasOwnProperty("responses")) {
            if(item.responses.hasOwnProperty(filter.key)) {
              item.properties[filter.key] = item.responses[filter.key];
            }
            // else {
            //   obj.properties[filter.key] = 'undefined';
            // }
          }
          // else {
          //   obj.properties[filter.key] = 'undefined';
          // }
        }

        // Project the object
        //if (self._projection !== mapProjection){
        //  return projector.project.Feature(self._projection, mapProjection, item);
        //}
        return item;
      }

      function addFeature(doc) {
        features.push(resultToGeoJSON(doc));
      }

      // ----------------------------------------------------------
      // Get the data
      // Set to true to try streaming
      // Streaming is both broken and not optimized
      if(false) {
        stream.on('data', addFeature);

        stream.on('close', function() {
          console.log("Fetched and processed " + features.length + " responses in " + (Date.now() - start) + "ms");

          callback(null, {
            type: 'FeatureCollection',
            features: features
          });
        }.bind(this));
      }else {

        // The same thing as above, except using toArray
        // NB -- conversion to a format that the renderer wants is broken
        cursor.toArray(function(error, results) {
          if(error) {
            console.log("Error fetching results: ", error);
          }

          if(results) {
            console.log("Fetched " + results.length + " responses in " + (Date.now() - start) + "ms");
          }

          start = Date.now();
          var len = results.length;

          // Convert the results to geojson
          for (var i = 0; i < len; i++) {
            results[i] = resultToGeoJSON(results[i]);
          }

          var fc = {
            type: 'FeatureCollection',
            features: results
          };

          if (self._projection !== mapProjection) {
            fc = projector.project.FeatureCollection(self._projection, mapProjection, fc);
          }

          console.log("Processed " + results.length + " responses in " + (Date.now() - start) + "ms");

          callback(null, fc);

        }.bind(this));

      }

      // Going to skip reprojecting
      // features.push(projectFeature(mapProjection, geoJSONDoc));
    }.bind(this));
  }
};
<commit_msg>Update to take db object instead of connection<commit_after>var MongoClient = require('mongodb').MongoClient;
var projector = require("nodetiles-core").projector;
var __ = require("lodash");


var MongoSource = function(options) {
  this._projection = projector.util.cleanProjString(options.projection || 'EPSG:4326');
  // this._connectionString = options.connectionString; // required
  this._db = options.db;
  this._collectionName = options.collectionName;     // required
  this._geoKey = options.key;                        // required - name of the geodata field
  this._query =   options.query;                     // required - a query to get started with
  this._select = options.select;                     // required - fields to select
  this._filter = options.filter;                     // optional - key to filter data
  this.sourceName = options.name || 'localdata';
  return this;
};

MongoSource.prototype = {
  constructor: MongoSource,

  getShapes: function(minX, minY, maxX, maxY, mapProjection, callback) {

    // First, we need to turn the coordinates into a bounds query for Mongo
    var min = [minX, minY];
    var max = [maxX, maxY];

    // project request coordinates into data coordinates
    if (mapProjection !== this._projection) {
      min = projector.project.Point(mapProjection, this._projection, min);
      max = projector.project.Point(mapProjection, this._projection, max);
    }

    var query = this._query || {};
    var parsedBbox = [[min[0], min[1]], [max[0],  max[1]]];
    query[this._geoKey] = { '$within': { '$box': parsedBbox } };

    // TODO: set autoreconnect
    //MongoClient.connect(this._connectionString, {
    //}, function(err, db) {
      //if(err) {
      //  console.log("Mongo error:", err);
      //  return;
      //}

      start = Date.now();
      // var stream = this._db.collection(this._collectionName)
      //   .find(this._query, this._select)
      //   .stream();

      var cursor = this._db.collection(this._collectionName)
        .find(this._query, this._select);

      // console.log("Query %j %j", this._query, this._select);

      var features = [];

      var self = this;

      /**
       * Turn stored a stored parcel result into geoJSON
       * TODO: Can save time and memory by not creating a new object here.
       * @param  {Object} item  A single response
       * @return {Object}       A single response structured as geoJSON
       */
      function resultToGeoJSON(item, filter) {
        item.type = 'Feature';

        // Get the shape
        // Or if there isn't one, use the centroid.
        if (item.geo_info.geometry !== undefined) {
          item.id = item.parcel_id;
          item.geometry = item.geo_info.geometry;
        }else {
          item.id = item._id;
          item.geometry = {};
          item.geometry.type = 'Point';
          item.geometry.coordinates = item.geo_info.centroid;
        }

        // item.properties = item;
        item.properties = {
          geometry: item.geo_info.geometry,
          name: item.geo_info.humanReadableName
        };

        // Clean up a bit
        delete item.geo_info.centroid;
        delete item.geo_info.geometry;
        delete item.responses;

        // If there is a filer, we also want the key easily accessible.
        if(filter) {
          // TODO: Handle the undefined condition
          if(item.hasOwnProperty("responses")) {
            if(item.responses.hasOwnProperty(filter.key)) {
              item.properties[filter.key] = item.responses[filter.key];
            }
            // else {
            //   obj.properties[filter.key] = 'undefined';
            // }
          }
          // else {
          //   obj.properties[filter.key] = 'undefined';
          // }
        }

        // Project the object
        //if (self._projection !== mapProjection){
        //  return projector.project.Feature(self._projection, mapProjection, item);
        //}
        return item;
      }

      function addFeature(doc) {
        features.push(resultToGeoJSON(doc));
      }

      // ----------------------------------------------------------
      // Get the data
      // Set to true to try streaming
      // Streaming is both broken and not optimized
      if(false) {
        stream.on('data', addFeature);

        stream.on('close', function() {
          console.log("Fetched and processed " + features.length + " responses in " + (Date.now() - start) + "ms");

          callback(null, {
            type: 'FeatureCollection',
            features: features
          });
        }.bind(this));
      }else {

        // The same thing as above, except using toArray
        // NB -- conversion to a format that the renderer wants is broken
        cursor.toArray(function(error, results) {
          if(error) {
            console.log("Error fetching results: ", error);
          }

          if(results) {
            console.log("Fetched " + results.length + " responses in " + (Date.now() - start) + "ms");
          }

          start = Date.now();
          var len = results.length;

          // Convert the results to geojson
          for (var i = 0; i < len; i++) {
            results[i] = resultToGeoJSON(results[i]);
          }

          var fc = {
            type: 'FeatureCollection',
            features: results
          };

          if (self._projection !== mapProjection) {
            fc = projector.project.FeatureCollection(self._projection, mapProjection, fc);
          }

          console.log("Processed " + results.length + " responses in " + (Date.now() - start) + "ms");

          callback(null, fc);

        }.bind(this));

      }

      // Going to skip reprojecting
      // features.push(projectFeature(mapProjection, geoJSONDoc));
    //}.bind(this));
  }
};

{'fraction_non_alphanumeric': 0.13350060564111438, 'fraction_numerical': 0.0015573628655476727, 'mean_word_length': 1.8668154761904763, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3648257', 'n_tokens_mistral': 3275, 'n_tokens_neox': 3149, 'n_words': 803}