<commit_before>#!/usr/bin/env node

// small benchmark to execute with nodejs

var http = require('http')
var url = require('url');
var fs = require('fs');
var me = process.argv[1];

function usage(exit_code) {
  console.log("Usage: " + me + " [OPTIONS] <example_tile_url>");
  console.log("Options:");
  console.log(" -v                      verbose operations (off)");
  console.log(" --help                  print this help");
  console.log(" --key <string>          map authentication key (none)");
  console.log(" -n, --requests <num>    number of requests to send (1000)");
  console.log(" -C, --cached <num>      number of requests sharing same cache id (20)");
  console.log(" -c, --concurrent <num>  number of concurrent requests (20)");
  console.log(" --vp-size <num>x<num>   tiles per col, line in viewport (5x4)");
  console.log(" --zoom-levels <num>     number of zoom levels to span");
  process.exit(exit_code);
}

process.argv.shift(); // this will be "node" (argv[0])
process.argv.shift(); // this will be the script name  (argv[1])

var verbose = 0;
var urltemplate;
var map_key;
var cached_requests = 20;
var N = cached_requests*50; // number of requests (50 full viewports)
var concurrency = cached_requests; // number of concurrent requests
var cols = 5;  
var lines = 4;
var zlevs = 2; 

var arg;
while ( arg = process.argv.shift() ) {
  if ( arg == '-v' ) {
    ++verbose;
  }
  else if ( arg == '--key' ) {
    map_key=process.argv.shift();
  }
  else if ( arg == '--cached' || arg == '-C' ) {
    cached_requests=process.argv.shift();
  }
  else if ( arg == '--requests' || arg == '-n' ) {
    N = parseInt(process.argv.shift());
  }
  else if ( arg == '--vp-size' ) {
    arg = process.argv.shift();
    var parsed = arg.match(/([0-9]*)x([0-9]*)/);
    cols = parseInt(parsed[1]);
    lines = parseInt(parsed[2]);
  }
  else if ( arg == '--zoom-levels' ) {
    zlevs = parseInt(process.argv.shift());
  }
  else if ( arg == '--help' ) {
    usage(0);
  }
  else if ( arg == '-c' || arg == '--concurrent' ) {
    concurrency = parseInt(process.argv.shift());
  }
  else if ( ! urltemplate ) {
    urltemplate = arg;
  }
  else {
    usage(1);
  }
}

if ( ! urltemplate ) {
  usage(1);
}

var urlparsed = url.parse(urltemplate, true);
urlparsed.query = urlparsed.query || {};
delete urlparsed.search; // or url.format will not use urlparsed.query
var pathname_match = urlparsed.pathname.match(RegExp('(.*)/[0-9]+/[0-9]+/[0-9]+.png$', "i"));
if ( ! pathname_match ) {
  // For backward compatibility, add ZXY portion to url, if not found
  urlparsed.pathname += '/{z}/{x}/{y}.png';
} else {
  // Otherwise take the whole thing as a sample and convert ZXY
  // with the templated version
  //urlparsed.pathname += '/{z}/{x}/{y}.png';
  urlparsed.pathname = pathname_match[1];
  urlparsed.pathname += '/{z}/{x}/{y}.png';
}

if ( map_key ) {
  urlparsed.query['map_key'] = map_key;
}

urltemplate = url.format(urlparsed);

var start_time = Date.now();
function end() {
    var end_time = Date.now();
    var t = (end_time - start_time)/1000;
    console.log("");
    console.log("Server Host:          ", urlparsed.host);
    console.log("Template URL (path):  ", urlparsed.pathname);
    console.log("");
    console.log("Viewport size:        " + cols + "x" + lines);
    console.log("Zoom levels:          " + zlevs);
    console.log("");
    console.log("Requests per cache:   ", cached_requests);
    console.log("Complete requests:    ", ok);
    console.log("");
    console.log("X-Cache hits:         ", xchits, " (" + Math.round((xchits/ok)*100) + "% of complete requests)" );
    console.log("X-Varnish hits:       ", xvhits, " (" + Math.round((xvhits/ok)*100) + "% of complete requests)" );
    console.log("Application hits:     ", ok-xchits-xvhits);
    console.log("Hits summary:         ", xchits, "-", xvhits, "-", (ok-xchits-xvhits));
    console.log("");
    console.log("Failed requests:      ", error);
    console.log("Concurrency Level:    ", concurrency);
    console.log("Time taken for tests: ", t, " seconds");
    console.log("Requests per second:  ", (Math.round(((ok+error)/t)*100)/100), '[#/sec] (mean)');
    console.log("");
    process.exit(0);
}

var ok = 0;
var xchits = 0; // X-Cache hits
var xvhits = 0; // X-Varnish hits
var error = 0;

function check_end() {
  if ( error + ok === N ) end();
}

function fail(msg) {
  console.log(msg);
  ++ error;
  check_end();
}

function pass() {
  ++ ok;
  check_end();
}

http.globalAgent.maxSockets = concurrency;
var now = Date.now();
var cbprefix = 'wb_' + process.env.USER + '_' + process.pid + "_"; 
for(var i = 0; i < N; ++i) {

    // we start at zoom level 3 (TODO: make configurable)
    var z = 3 + Math.floor( (i/(cols*lines))%zlevs ); 
    var x = i%cols;
    var y = Math.floor(i/cols)%lines;

    // update cache_buster every "cached_requests" requests 
    now += Math.floor(i/cached_requests);
    var cb = cbprefix + now;

    var nurlobj = url.parse(urltemplate, true);
    delete nurlobj.search; // or url.format will not use urlparsed.query
    nurlobj.pathname = nurlobj.pathname.replace('{z}', z).replace('{x}', x).replace('{y}', y);
    nurlobj.query = nurlobj.query || {};
    nurlobj.query['cache_buster'] = cb;

    var nurl = url.format(nurlobj);

    if ( verbose ) console.log("|+++|---> " + nurl);

    //console.log(opt.path)
    http.get(nurl, function(res) {
      res.body = '';
      if ( res.statusCode != 200 ) {
        res.on('data', function(chunk) {
          // Save only first chunk, to reduce cost of the operation
          if ( res.body.length == 0 ) res.body += chunk;
        });
      }<commit_msg>Add a --grid switch to fetch an utf8 grid for each tile<commit_after>#!/usr/bin/env node

// small benchmark to execute with nodejs

var http = require('http')
var url = require('url');
var fs = require('fs');
var me = process.argv[1];

function usage(exit_code) {
  console.log("Usage: " + me + " [OPTIONS] <example_tile_url>");
  console.log("Options:");
  console.log(" -v                      verbose operations (off)");
  console.log(" --help                  print this help");
  console.log(" --key <string>          map authentication key (none)");
  console.log(" -n, --requests <num>    number of requests to send (1000)");
  console.log(" -C, --cached <num>      number of requests sharing same cache id (20)");
  console.log(" -c, --concurrent <num>  number of concurrent requests (20)");
  console.log(" --vp-size <num>x<num>   tiles per col, line in viewport (5x4)");
  console.log(" --zoom-levels <num>     number of zoom levels to span");
  console.log(" --grid                  also fetch an utf8grid for each tile");
  process.exit(exit_code);
}

process.argv.shift(); // this will be "node" (argv[0])
process.argv.shift(); // this will be the script name  (argv[1])

var verbose = 0;
var urltemplate;
var map_key;
var cached_requests = 20;
var N = cached_requests*50; // number of requests (50 full viewports)
var concurrency = cached_requests; // number of concurrent requests
var cols = 5;  
var lines = 4;
var zlevs = 2; 
var fetch_grid = false;

var arg;
while ( arg = process.argv.shift() ) {
  if ( arg == '-v' ) {
    ++verbose;
  }
  else if ( arg == '--key' ) {
    map_key=process.argv.shift();
  }
  else if ( arg == '--cached' || arg == '-C' ) {
    cached_requests=process.argv.shift();
  }
  else if ( arg == '--grid' ) {
    fetch_grid=true;
  }
  else if ( arg == '--requests' || arg == '-n' ) {
    N = parseInt(process.argv.shift());
  }
  else if ( arg == '--vp-size' ) {
    arg = process.argv.shift();
    var parsed = arg.match(/([0-9]*)x([0-9]*)/);
    cols = parseInt(parsed[1]);
    lines = parseInt(parsed[2]);
  }
  else if ( arg == '--zoom-levels' ) {
    zlevs = parseInt(process.argv.shift());
  }
  else if ( arg == '--help' ) {
    usage(0);
  }
  else if ( arg == '-c' || arg == '--concurrent' ) {
    concurrency = parseInt(process.argv.shift());
  }
  else if ( ! urltemplate ) {
    urltemplate = arg;
  }
  else {
    usage(1);
  }
}

if ( ! urltemplate ) {
  usage(1);
}

var urlparsed = url.parse(urltemplate, true);
urlparsed.query = urlparsed.query || {};
delete urlparsed.search; // or url.format will not use urlparsed.query
var pathname_match = urlparsed.pathname.match(RegExp('(.*)/[0-9]+/[0-9]+/[0-9]+.png$', "i"));
if ( ! pathname_match ) {
  // For backward compatibility, add ZXY portion to url, if not found
  urlparsed.pathname += '/{z}/{x}/{y}.png';
} else {
  // Otherwise take the whole thing as a sample and convert ZXY
  // with the templated version
  //urlparsed.pathname += '/{z}/{x}/{y}.png';
  urlparsed.pathname = pathname_match[1];
  urlparsed.pathname += '/{z}/{x}/{y}.png';
}

if ( map_key ) {
  urlparsed.query['map_key'] = map_key;
}

urltemplate = url.format(urlparsed);

var start_time = Date.now();
function end() {
    var end_time = Date.now();
    var t = (end_time - start_time)/1000;
    console.log("");
    console.log("Server Host:          ", urlparsed.host);
    console.log("Template URL (path):  ", urlparsed.pathname);
    console.log("");
    console.log("Viewport size:        " + cols + "x" + lines);
    console.log("Zoom levels:          " + zlevs);
    console.log("");
    console.log("Requests per cache:   ", cached_requests);
    console.log("Complete requests:    ", ok);
    console.log("");
    console.log("X-Cache hits:         ", xchits, " (" + Math.round((xchits/ok)*100) + "% of complete requests)" );
    console.log("X-Varnish hits:       ", xvhits, " (" + Math.round((xvhits/ok)*100) + "% of complete requests)" );
    console.log("Application hits:     ", ok-xchits-xvhits);
    console.log("Hits summary:         ", xchits, "-", xvhits, "-", (ok-xchits-xvhits));
    console.log("");
    console.log("Failed requests:      ", error);
    console.log("Concurrency Level:    ", concurrency);
    console.log("Time taken for tests: ", t, " seconds");
    console.log("Requests per second:  ", (Math.round(((ok+error)/t)*100)/100), '[#/sec] (mean)');
    console.log("");
    process.exit(0);
}

var ok = 0;
var xchits = 0; // X-Cache hits
var xvhits = 0; // X-Varnish hits
var error = 0;

function check_end() {
  if ( error + ok === N ) end();
}

function fail(msg) {
  console.log(msg);
  ++ error;
  check_end();
}

function pass() {
  ++ ok;
  check_end();
}

http.globalAgent.maxSockets = concurrency;
var now = Date.now();
var cbprefix = 'wb_' + process.env.USER + '_' + process.pid + "_"; 
for(var i = 0; i < N; ++i) {

    // we start at zoom level 3 (TODO: make configurable)
    var z = 3 + Math.floor( (i/(cols*lines))%zlevs ); 
    var x = i%cols;
    var y = Math.floor(i/cols)%lines;

    // update cache_buster every "cached_requests" requests 
    now += Math.floor(i/cached_requests);
    var cb = cbprefix + now;

    var nurlobj = url.parse(urltemplate, true);
    delete nurlobj.search; // or url.format will not use urlparsed.query
    nurlobj.pathname = nurlobj.pathname.replace('{z}', z).replace('{x}', x).replace('{y}', y);
    if ( fetch_grid && i%2 ) {
      nurlobj.pathname = nurlobj.pathname.replace('.png', '.grid.json');
    }
    nurlobj.query = nurlobj.query || {};
    nurlobj.query['cache_buster'] = cb;

    var nurl = url.format(nurlobj);

    if ( verbose ) console.log("|+++|---> " + nurl);

    //console.log(opt.path)
    http.get(nurl, function(res) {
      res.body = '';
      if ( res.statusCode != 200 ) {
        res.on('data', function(chunk) {
          // Save only first chunk, to reduce cost of the operation
          if ( res.body.length == 0 ) res.body += chunk;
        });
      }
{'fraction_non_alphanumeric': 0.18785006462731582, 'fraction_numerical': 0.011805256355019389, 'mean_word_length': 2.8596607914865313, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7033078', 'n_tokens_mistral': 4105, 'n_tokens_neox': 4053, 'n_words': 1010}