<commit_before>var fs = require('fs');
var express = require('express');
var async = require('async');
var topojson = require('topojson');
var pg = require('pg');
var SphericalMercator = require('@mapbox/sphericalmercator');
var Caching = require('caching');
var clone = require('clone');
var connectTimeout = require('connect-timeout');
var mapnik = require('mapnik');
var path = require('path');

mapnik.register_datasource(path.join(mapnik.settings.paths.input_plugins, 'geojson.input'));

function setStatementTimeout(client) {
  return new Promise(function(resolve, reject){
    var statementTimeout = parseInt(process.env.TILESPLASH_STATEMENT_TIMEOUT) || 0;
    if (statementTimeout && statementTimeout > 0) {
      client.query('SET statement_timeout='+statementTimeout, [], function(err){
        if (err) {
          reject(err);
        } else {
          resolve();
        }
      });
    } else {
      resolve();
    }
  });
}

// allowing tho options object for addGeoJson() - simplification settings, etc.
function stringifyProtobuf(layers, tile, mvtOptions) {
  var vtile = new mapnik.VectorTile(tile.z, tile.x, tile.y);
  for (var layerName in layers) {
    vtile.addGeoJSON(JSON.stringify(layers[layerName]), layerName, mvtOptions);
  }
  return vtile.getData();
}

var pgMiddleware = function(_dbOptions){
  var dbOptions;
  if (typeof _dbOptions === 'string') {
    dbOptions = _dbOptions;
  } else {
    dbOptions = {};
    Object.keys(_dbOptions).forEach(function(key){
      dbOptions[key] = _dbOptions[key];
    });
    dbOptions.poolSize = dbOptions.poolSize || 64;
  }

  return function(req, res, next){
    req.db = {};
    req.db.query = function(sql, bindvars, callback){
      if (typeof bindvars === 'function') {
        callback = bindvars;
        bindvars = undefined;
      }

      var poolTimedOut = false;
      var poolTimeout = setTimeout(function(){
        poolTimedOut = true;
        callback(new Error('failed to get db connection'));
      }, 5000);

      pg.connect(dbOptions, function(err, client, done){
        if (poolTimedOut) {
          done();
          return;
        }

        clearTimeout(poolTimeout);
        if (err) {
          callback(err);
          done();
          return;
        }

        res.on('finish', done);
        res.on('error', done);
        req.on('timeout', done);

        setStatementTimeout(client).then(function(){
          client.query(sql, bindvars, function(err, result){<commit_msg>use pool.connect<commit_after>var fs = require('fs');
var express = require('express');
var async = require('async');
var topojson = require('topojson');
var pg = require('pg');
var SphericalMercator = require('@mapbox/sphericalmercator');
var Caching = require('caching');
var clone = require('clone');
var connectTimeout = require('connect-timeout');
var mapnik = require('mapnik');
var path = require('path');

mapnik.register_datasource(path.join(mapnik.settings.paths.input_plugins, 'geojson.input'));

function setStatementTimeout(client) {
  return new Promise(function(resolve, reject){
    var statementTimeout = parseInt(process.env.TILESPLASH_STATEMENT_TIMEOUT) || 0;
    if (statementTimeout && statementTimeout > 0) {
      client.query('SET statement_timeout='+statementTimeout, [], function(err){
        if (err) {
          reject(err);
        } else {
          resolve();
        }
      });
    } else {
      resolve();
    }
  });
}

// allowing tho options object for addGeoJson() - simplification settings, etc.
function stringifyProtobuf(layers, tile, mvtOptions) {
  var vtile = new mapnik.VectorTile(tile.z, tile.x, tile.y);
  for (var layerName in layers) {
    vtile.addGeoJSON(JSON.stringify(layers[layerName]), layerName, mvtOptions);
  }
  return vtile.getData();
}

var pgMiddleware = function(_dbOptions){
  var dbOptions;
  if (typeof _dbOptions === 'string') {
    dbOptions = _dbOptions;
  } else {
    dbOptions = {};
    Object.keys(_dbOptions).forEach(function(key){
      dbOptions[key] = _dbOptions[key];
    });
    dbOptions.poolSize = dbOptions.poolSize || 64;
  }

  return function(req, res, next){
    req.db = {};
    req.db.query = function(sql, bindvars, callback){
      if (typeof bindvars === 'function') {
        callback = bindvars;
        bindvars = undefined;
      }

      var poolTimedOut = false;
      var poolTimeout = setTimeout(function(){
        poolTimedOut = true;
        callback(new Error('failed to get db connection'));
      }, 5000);

      var pool = new pg.Pool();
      
      pool.connect(dbOptions, function(err, client, done){
        if (poolTimedOut) {
          done();
          return;
        }

        clearTimeout(poolTimeout);
        if (err) {
          callback(err);
          done();
          return;
        }

        res.on('finish', done);
        res.on('error', done);
        req.on('timeout', done);

        setStatementTimeout(client).then(function(){
          client.query(sql, bindvars, function(err, result){
{'fraction_non_alphanumeric': 0.1410871302957634, 'fraction_numerical': 0.0031974420463629096, 'mean_word_length': 3.029790660225443, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2790762', 'n_tokens_mistral': 1506, 'n_tokens_neox': 1479, 'n_words': 332}