<commit_before>var Promise = require('bluebird');
var pgp = require('pg-promise')({ promiseLib: Promise });
var fs = require('fs');
var path = require('path');
var sprintf = require('sprintf-js').sprintf;
var exec = require('child-process-promise').exec;
var QueryStream = require('pg-query-stream');
var JSONStream = require('JSONStream');

var crawler = require('../crawler/crawler.js');
var params = require('./params.js');
var db = pgp(params.dbConnStr);
var source = sprintf('%s/regions.geojson', path.resolve(__dirname, params.tempDir));
var target = sprintf('%s/regions.mbtiles', path.resolve(__dirname, params.tileDir));

var sql = `
  WITH info AS (
   SELECT
     vii.region_id AS id,
     SUM(vii.count) AS count,
     array_agg(json_build_object(
       'id', vii.instance_id,
       'name', vii.instance_name,
       'count', vii.count,
       'update', vii.update_date,
       'topTag', vii.tags[1],
       'topOrganization', vii.organizations[1],
       'topCategory', vii.categories[1]
     )) AS instances
   FROM instance_region_xref AS irx
     LEFT JOIN view_instance_info as vii
       ON vii.instance_id = irx.instance_id AND vii.region_id = irx.region_id
     LEFT JOIN instance AS i ON i.id = irx.instance_id
   WHERE i.active
   GROUP BY vii.region_name, vii.region_id
  )
  SELECT
   'Feature' AS type,
   json_build_object('maxzoom', rl.max_tile_zoom, 'minzoom', rl.min_tile_zoom) AS tippecanoe,
   ST_AsGeoJSON(ST_Simplify(r.geom, 0.001), 3)::json AS geometry,
   json_build_object(
     'id', r.id,
     'name', r.name,
     'bbox', ST_AsGeoJSON(r.bbox, 3),
     'count', info.count,
     'instances', info.instances
   ) AS properties
  FROM info, region AS r
   LEFT JOIN region_level AS rl ON rl.id = r.region_level_id
  WHERE r.id = info.id AND r.geom IS NOT NULL
  ORDER BY rl.id
`;
<commit_msg>fix a bug at tile generation<commit_after>var Promise = require('bluebird');
var pgp = require('pg-promise')({ promiseLib: Promise });
var fs = require('fs');
var path = require('path');
var sprintf = require('sprintf-js').sprintf;
var exec = require('child-process-promise').exec;
var QueryStream = require('pg-query-stream');
var JSONStream = require('JSONStream');

var crawler = require('../crawler/crawler.js');
var params = require('./params.js');
var db = pgp(params.dbConnStr);
var source = sprintf('%s/regions.geojson', path.resolve(__dirname, params.tempDir));
var target = sprintf('%s/regions.mbtiles', path.resolve(__dirname, params.tileDir));

var sql = `
  WITH info AS (
   SELECT
     vii.region_id AS id,
     SUM(vii.count) AS count,
     array_agg(json_build_object(
       'id', vii.instance_id,
       'name', vii.instance_name,
       'count', vii.count,
       'update', vii.update_date,
       'topTag', vii.tags[1],
       'topOrganization', vii.organizations[1],
       'topCategory', vii.categories[1]
     )) AS instances
   FROM instance_region_xref AS irx
     JOIN view_instance_info as vii
       ON vii.instance_id = irx.instance_id AND vii.region_id = irx.region_id
     LEFT JOIN instance AS i ON i.id = irx.instance_id
   WHERE i.active
   GROUP BY vii.region_name, vii.region_id
  )
  SELECT
   'Feature' AS type,
   json_build_object('maxzoom', rl.max_tile_zoom, 'minzoom', rl.min_tile_zoom) AS tippecanoe,
   ST_AsGeoJSON(ST_Simplify(r.geom, 0.001), 3)::json AS geometry,
   json_build_object(
     'id', r.id,
     'name', r.name,
     'bbox', ST_AsGeoJSON(r.bbox, 3),
     'count', info.count,
     'instances', info.instances
   ) AS properties
  FROM info, region AS r
   LEFT JOIN region_level AS rl ON rl.id = r.region_level_id
  WHERE r.id = info.id AND r.geom IS NOT NULL
  ORDER BY rl.id
`;

{'fraction_non_alphanumeric': 0.13740458015267176, 'fraction_numerical': 0.004907306434023991, 'mean_word_length': 3.6150943396226416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '214040', 'n_tokens_mistral': 1332, 'n_tokens_neox': 1299, 'n_words': 316}