<commit_before>var terraformer = require('terraformer'),
  sql = require('sql-parser'),
  _ = require('lodash');

module.exports = {
  // process all query params filters
  filter: function( json, params, callback ){
    if ( params.geometry ){
      this.geometryFilter( json, params, callback );
    } else if ( params.where ){
      this.whereFilter( json, params, callback );
    } else {
      //probably better to parse all params upfront and confirm valid types
      if ( params.returnCountOnly === 'true' || params.returnCountOnly ){
        
        callback( null, { count: json.features.length });

      } else if ( params.returnIdsOnly === 'true' || params.returnIdsOnly){
        this.getIds( json, params.idField, params, function( err, ids ){
          callback( err, ids );
        });

      } else {

        if (params.orderByFields && params.orderByFields !== ''){
          var fld = params.orderByFields.split(' ');
          var order = params.orderByFields;
          if (fld[fld.length-1] == 'DESC'){
            order = '-' + params.orderByFields;
          }
          json.features = this.orderBy(json.features, params.orderByFields, order);
        }

        if ( ( params.returnGeometry === false || params.returnGeometry == 'false') && (!params.outFields || params.outFields != '*')){
          json.features.forEach(function(f){
            delete f.geometry;
          });
        } else if ( params.outSR && ( params.outSR == '102100' ) ){
          if ( json.spatialReference ){
            json.spatialReference.wkid = params.outSR;
            var coords;
            // project each geometry to merator
            json.features.forEach( function( f ){
              if ( f.geometry.x && f.geometry.y ) {
                coords = new terraformer.Point( [f.geometry.x, f.geometry.y] ).toMercator().coordinates;
                f.geometry.x = coords[0];
                f.geometry.y = coords[1];
              } else if ( f.geometry.rings ){
                f.geometry.rings = new terraformer.Polygon( f.geometry.rings ).toMercator().coordinates;
              } else if ( f.geometry.paths ) {
                f.geometry.paths = new terraformer.LineString( f.geometry.paths ).toMercator().coordinates;
              }
              f.geometry.spatialReference.wkid = params.outSR;
            });
          }
        }

        // checkout for outfields
        if ( params.outFields && params.outFields != '*' ){
          var features = [],
            props,
            outFields = params.outFields.split( ',' );

          json.features.forEach( function( f ){
            props = _.pick(f.attributes || f.properties, outFields);

            var newFeature = { geometry: f.geometry};
            if ( f.properties ) {
              newFeature.properties = props;
            } else {
              newFeature.attributes = props;
            }
            features.push( newFeature );
          });
          json.features = features;
        }

        // before we send back json, process outStats
        if ( params.outStatistics ){
          this.outStatistics( json, params, callback );
        } else {
          callback( null, json );
        }


      }

    }

  },

  calculateStat: function(type, field, features){
    var propName = (features[0].attributes) ? 'attributes' : 'properties';
    var types = {
      'min': function(field, features){
        var min = features[0][propName][field];
        features.forEach(function(f,i){
          if (f[propName][field] < min){
            min = f[propName][field];
          }
        });
        return min;
      },
      'max': function(field, features){
        var max = features[0][propName][field];
        features.forEach(function(f,i){
          if (f[propName][field] > max){
            max = f[propName][field];
          }
        });
        return max;
      },
      'count': function(field, features){
        var count = 0;
        features.forEach(function(f,i){
          if (f[propName][field]){
            count++;
          }
        });
        return count;
      },
      'sum': function(field, features){
        var sum = 0;
        features.forEach(function(f,i){
          if (f[propName][field]){
            sum += parseFloat(f[propName][field]);
          }
        });
        return sum;
      },
      'avg': function(field, features){
        var sum = types.sum(field, features);
        return sum/features.length;
      },
      'stddev': function(field, features){
        var v = types.var( field, features );
        return Math.sqrt(v);
      },
      'var': function(field, features){
        var avg = types.avg( field, features ),
          i = features.length,
          v = 0;

        while( i-- ){
          v += Math.pow( (features[ i ][propName][field] - avg), 2 );
        }
        v /= features.length;
        return v;
      }
    };
    //count | sum | avg | stddev | var
    return types[type](field, features);
  },

  outStatistics: function( json, params, callback ){
    var result = {}, value, self = this;
    try {
      json.fields = [];
      var stats = JSON.parse(params.outStatistics);
      if (stats.length){

        if ( params.groupByFieldsForStatistics ){
          this.buildUniqueFeatures(params, json.features, function( err, uniques ){
            if (err){
              return callback("unable to build unique features with property " + params.groupByFieldsForStatistics, null); 
            }
            var statCollection = [];
            var finalJson = {fields:null, features:[]};

            for ( var value in uniques ){
              var statResult = self.buildStats( stats, { features: uniques[ value ] }, params );
              // always has one feature, must add the value of grouped attr
              statResult.features[0].attributes[ params.groupByFieldsForStatistics ] = value;
              finalJson.fields = statResult.fields;
              finalJson.features.push( statResult.features[0] );
            }

            // create a field entry for the grouped attribute
            finalJson.fields.push({
              name: params.groupByFieldsForStatistics,
              type: self.fieldType( finalJson.features[1].attributes[ params.groupByFieldsForStatistics ] ),<commit_msg>supporting upper case stat types<commit_after>var terraformer = require('terraformer'),
  sql = require('sql-parser'),
  _ = require('lodash');

module.exports = {
  // process all query params filters
  filter: function( json, params, callback ){
    if ( params.geometry ){
      this.geometryFilter( json, params, callback );
    } else if ( params.where ){
      this.whereFilter( json, params, callback );
    } else {
      //probably better to parse all params upfront and confirm valid types
      if ( params.returnCountOnly === 'true' || params.returnCountOnly ){
        
        callback( null, { count: json.features.length });

      } else if ( params.returnIdsOnly === 'true' || params.returnIdsOnly){
        this.getIds( json, params.idField, params, function( err, ids ){
          callback( err, ids );
        });

      } else {

        if (params.orderByFields && params.orderByFields !== ''){
          var fld = params.orderByFields.split(' ');
          var order = params.orderByFields;
          if (fld[fld.length-1] == 'DESC'){
            order = '-' + params.orderByFields;
          }
          json.features = this.orderBy(json.features, params.orderByFields, order);
        }

        if ( ( params.returnGeometry === false || params.returnGeometry == 'false') && (!params.outFields || params.outFields != '*')){
          json.features.forEach(function(f){
            delete f.geometry;
          });
        } else if ( params.outSR && ( params.outSR == '102100' ) ){
          if ( json.spatialReference ){
            json.spatialReference.wkid = params.outSR;
            var coords;
            // project each geometry to merator
            json.features.forEach( function( f ){
              if ( f.geometry.x && f.geometry.y ) {
                coords = new terraformer.Point( [f.geometry.x, f.geometry.y] ).toMercator().coordinates;
                f.geometry.x = coords[0];
                f.geometry.y = coords[1];
              } else if ( f.geometry.rings ){
                f.geometry.rings = new terraformer.Polygon( f.geometry.rings ).toMercator().coordinates;
              } else if ( f.geometry.paths ) {
                f.geometry.paths = new terraformer.LineString( f.geometry.paths ).toMercator().coordinates;
              }
              f.geometry.spatialReference.wkid = params.outSR;
            });
          }
        }

        // checkout for outfields
        if ( params.outFields && params.outFields != '*' ){
          var features = [],
            props,
            outFields = params.outFields.split( ',' );

          json.features.forEach( function( f ){
            props = _.pick(f.attributes || f.properties, outFields);

            var newFeature = { geometry: f.geometry};
            if ( f.properties ) {
              newFeature.properties = props;
            } else {
              newFeature.attributes = props;
            }
            features.push( newFeature );
          });
          json.features = features;
        }

        // before we send back json, process outStats
        if ( params.outStatistics ){
          this.outStatistics( json, params, callback );
        } else {
          callback( null, json );
        }


      }

    }

  },

  calculateStat: function(type, field, features){
    var propName = (features[0].attributes) ? 'attributes' : 'properties';
    var types = {
      'min': function(field, features){
        var min = features[0][propName][field];
        features.forEach(function(f,i){
          if (f[propName][field] < min){
            min = f[propName][field];
          }
        });
        return min;
      },
      'max': function(field, features){
        var max = features[0][propName][field];
        features.forEach(function(f,i){
          if (f[propName][field] > max){
            max = f[propName][field];
          }
        });
        return max;
      },
      'count': function(field, features){
        var count = 0;
        features.forEach(function(f,i){
          if (f[propName][field]){
            count++;
          }
        });
        return count;
      },
      'sum': function(field, features){
        var sum = 0;
        features.forEach(function(f,i){
          if (f[propName][field]){
            sum += parseFloat(f[propName][field]);
          }
        });
        return sum;
      },
      'avg': function(field, features){
        var sum = types.sum(field, features);
        return sum/features.length;
      },
      'stddev': function(field, features){
        var v = types.var( field, features );
        return Math.sqrt(v);
      },
      'var': function(field, features){
        var avg = types.avg( field, features ),
          i = features.length,
          v = 0;

        while( i-- ){
          v += Math.pow( (features[ i ][propName][field] - avg), 2 );
        }
        v /= features.length;
        return v;
      }
    };
    //count | sum | avg | stddev | var
    return types[type.toLowerCase()](field, features);
  },

  outStatistics: function( json, params, callback ){
    var result = {}, value, self = this;
    try {
      json.fields = [];
      var stats = JSON.parse(params.outStatistics);
      if (stats.length){

        if ( params.groupByFieldsForStatistics ){
          this.buildUniqueFeatures(params, json.features, function( err, uniques ){
            if (err){
              return callback("unable to build unique features with property " + params.groupByFieldsForStatistics, null); 
            }
            var statCollection = [];
            var finalJson = {fields:null, features:[]};

            for ( var value in uniques ){
              var statResult = self.buildStats( stats, { features: uniques[ value ] }, params );
              // always has one feature, must add the value of grouped attr
              statResult.features[0].attributes[ params.groupByFieldsForStatistics ] = value;
              finalJson.fields = statResult.fields;
              finalJson.features.push( statResult.features[0] );
            }

            // create a field entry for the grouped attribute
            finalJson.fields.push({
              name: params.groupByFieldsForStatistics,
              type: self.fieldType( finalJson.features[1].attributes[ params.groupByFieldsForStatistics ] ),
{'fraction_non_alphanumeric': 0.13079299691040164, 'fraction_numerical': 0.003010377881644617, 'mean_word_length': 1.8470906630581867, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1816369', 'n_tokens_mistral': 3471, 'n_tokens_neox': 3396, 'n_words': 800}