<commit_before>var crypto = require('crypto');
var step = require('step');
var fs = require('fs');
var _ = require('underscore');
var PSQL = require('cartodb-psql');
var spawn = require('child_process').spawn;
var assert = require('assert');

// Keeps track of what's waiting baking for export
var bakingExports = {};

function OgrFormat(id) {
  this.id = id;
}

OgrFormat.prototype = {

  id: "ogr",

  is_file: true,

  getQuery: function(/*sql, options*/) {
    return null; // dont execute the query
  },

  transform: function(/*result, options, callback*/) {
    throw "should not be called for file formats";
  },

  getContentType: function(){ return this._contentType; },

  getFileExtension: function(){ return this._fileExtension; },

  getKey: function(options) {
    return [this.id,
        options.dbopts.dbname,
        options.dbopts.user,
        options.gn,
        this.generateMD5(options.filename),
        this.generateMD5(options.sql)].concat(options.skipfields).join(':');
  },

  generateMD5: function (data){
    var hash = crypto.createHash('md5');
    hash.update(data);
    return hash.digest('hex');
  }

};

// Internal function usable by all OGR-driven outputs
OgrFormat.prototype.toOGR = function(options, out_format, out_filename, callback) {

  //var gcol = options.gn;
  var sql = options.sql;
  var skipfields = options.skipfields;
  var out_layername = options.filename;

  var dbopts = options.dbopts;

  var ogr2ogr = global.settings.ogr2ogrCommand || 'ogr2ogr';
  var dbhost = dbopts.host;
  var dbport = dbopts.port;
  var dbuser = dbopts.user;
  var dbpass = <PASSWORD>;
  var dbname = dbopts.dbname;

  var timeout = options.timeout;

  var that = this;

  var columns = [];
  var geocol;
  var pg;

  // Drop ending semicolon (ogr doens't like it)
  sql = sql.replace(/;\s*$/, '');

  step (

    function fetchColumns() {
      var colsql = 'SELECT * FROM (' + sql + ') as _cartodbsqlapi LIMIT 0';
      pg = new PSQL(dbopts);
      pg.query(colsql, this);
    },
    // jshint maxcomplexity:9
    function findSRS(err, result) {
      assert.ifError(err);

      //if ( ! result.rows.length ) throw new Error("Query returns no rows");

      var needSRS = that._needSRS;

      // Skip system columns, find geom column
      for (var i=0; i<result.fields.length; ++i) {
        var field = result.fields[i];
        var k = field.name;
        if ( skipfields.indexOf(k) !== -1 ) {
            continue;
        }
        if ( out_format !== 'CSV' && k === "the_geom_webmercator" ) {
            continue;
        } // TODO: drop ?
        if ( out_format === 'CSV' ) {
            columns.push(pg.quoteIdentifier(k)+'::text');
        } else {
            columns.push(pg.quoteIdentifier(k));
        }

        if ( needSRS ) {
          if ( ! geocol && pg.typeName(field.dataTypeID) === 'geometry' ) {
            geocol = k;
          }
        }
      }
      //console.log(columns.join(','));

      if ( ! needSRS || ! geocol ) {
          return null;
      }

      var next = this;

      var qgeocol = pg.quoteIdentifier(geocol);
      var sridsql = 'SELECT ST_Srid(' + qgeocol + ') as srid, GeometryType(' +
                   qgeocol + ') as type FROM (' + sql + ') as _cartodbsqlapi WHERE ' +
                   qgeocol + ' is not null limit 1';

      pg.query(sridsql, function(err, result) {
        if ( err ) { next(err); return; }
        if ( result.rows.length ) {
          var srid = result.rows[0].srid;
          var type = result.rows[0].type;
          next(null, srid, type);
        } else {
          // continue as srid and geom type are not critical when there are no results
          next(null);
        }
      });

    },
    function spawnDumper(err, srid, type) {<commit_msg>Order geo-columns to export 'the_geom' by default while exporting SHP format<commit_after>var crypto = require('crypto');
var step = require('step');
var fs = require('fs');
var _ = require('underscore');
var PSQL = require('cartodb-psql');
var spawn = require('child_process').spawn;
var assert = require('assert');

// Keeps track of what's waiting baking for export
var bakingExports = {};

function OgrFormat(id) {
  this.id = id;
}

OgrFormat.prototype = {

  id: "ogr",

  is_file: true,

  getQuery: function(/*sql, options*/) {
    return null; // dont execute the query
  },

  transform: function(/*result, options, callback*/) {
    throw "should not be called for file formats";
  },

  getContentType: function(){ return this._contentType; },

  getFileExtension: function(){ return this._fileExtension; },

  getKey: function(options) {
    return [this.id,
        options.dbopts.dbname,
        options.dbopts.user,
        options.gn,
        this.generateMD5(options.filename),
        this.generateMD5(options.sql)].concat(options.skipfields).join(':');
  },

  generateMD5: function (data){
    var hash = crypto.createHash('md5');
    hash.update(data);
    return hash.digest('hex');
  }

};

// Internal function usable by all OGR-driven outputs
OgrFormat.prototype.toOGR = function(options, out_format, out_filename, callback) {

  //var gcol = options.gn;
  var sql = options.sql;
  var skipfields = options.skipfields;
  var out_layername = options.filename;

  var dbopts = options.dbopts;

  var ogr2ogr = global.settings.ogr2ogrCommand || 'ogr2ogr';
  var dbhost = dbopts.host;
  var dbport = dbopts.port;
  var dbuser = dbopts.user;
  var dbpass = dbopts.pass;
  var dbname = dbopts.dbname;

  var timeout = options.timeout;

  var that = this;

  var columns = [];
  var geocol;
  var pg;
  // Drop ending semicolon (ogr doens't like it)
  sql = sql.replace(/;\s*$/, '');

  const theGeomFirst = (fieldA, fieldB) => {
    if (fieldA.name === 'the_geom') {
      return -1;
    }
    if (fieldB.name === 'the_geom') {
      return 1;
    }
    return 0;
  };

  step (

    function fetchColumns() {
      var colsql = 'SELECT * FROM (' + sql + ') as _cartodbsqlapi LIMIT 0';
      pg = new PSQL(dbopts);
      pg.query(colsql, this);
    },
    function findSRS(err, result) {
      assert.ifError(err);

      var needSRS = that._needSRS;

      columns = result.fields
        // skip columns
        .filter(field => skipfields.indexOf(field.name) === -1)
        // put "the_geom" first (if exists)
        .sort(theGeomFirst)
        // get first geometry to calculate SRID ("the_geom" if exists)
        .map(field => {
          if (needSRS && !geocol && pg.typeName(field.dataTypeID) === 'geometry') {
            geocol = field.name;
          }

          return field;
        })
        // apply quotes to columns
        .map(field => out_format === 'CSV' ? pg.quoteIdentifier(field.name)+'::text' : pg.quoteIdentifier(field.name));

      if ( ! needSRS || ! geocol ) {
          return null;
      }

      var next = this;

      var qgeocol = pg.quoteIdentifier(geocol);
      var sridsql = 'SELECT ST_Srid(' + qgeocol + ') as srid, GeometryType(' +
                   qgeocol + ') as type FROM (' + sql + ') as _cartodbsqlapi WHERE ' +
                   qgeocol + ' is not null limit 1';

      pg.query(sridsql, function(err, result) {
        if ( err ) { next(err); return; }
        if ( result.rows.length ) {
          var srid = result.rows[0].srid;
          var type = result.rows[0].type;
          next(null, srid, type);
        } else {
          // continue as srid and geom type are not critical when there are no results
          next(null);
        }
      });
    },
    function spawnDumper(err, srid, type) {
{'fraction_non_alphanumeric': 0.1390877751259613, 'fraction_numerical': 0.0038451339167329623, 'mean_word_length': 2.4255222524977293, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3362528', 'n_tokens_mistral': 2480, 'n_tokens_neox': 2385, 'n_words': 617}