<commit_before>'use strict';

var linematch = require('linematch');
var lineclip = require('lineclip');

module.exports = function(data, tile, done) {

  // filter and normalize input geometry
  var tiger = toLines(data.tiger.tiger2015);
  var streets = toLines(data.osm.osm);

  // find tiger parts that are not covered by streets within 10 pixels
  var diff = linematch(tiger, streets, 10);

  if (diff.length) {
    // write a feature with the diff as MultiLineString
    process.stdout.write(',' + JSON.stringify({
      type: 'Feature',
      properties: {},
      geometry: {
        type: 'MultiLineString',
        coordinates: toGeoJSON(diff, tile)
      }
    }));
  }

  done(null, null);
};

function toGeoJSON(diff, tile) {
  var size = 4096 * Math.pow(2, tile[2]);
  var x0 = 4096 * tile[0];
  var y0 = 4096 * tile[1];

  for (var i = 0; i < diff.length; i++) {
    for (var j = 0; j < diff[i].length; j++) {
      var p = diff[i][j];
      diff[i][j] = [
        round((p[0] + x0) * 360 / size - 180),
        round(360 / Math.PI * Math.atan(Math.exp((180 - (p[1] + y0) * 360 / size) * Math.PI / 180)) - 90)
      ];
    }
  }
  return diff;
}

function round(num) {
  return Math.round(num * 1e6) / 1e6;
}

function toLines(layer) {
  var lines = [];
  var bbox = [0, 0, 4096, 4096];

  for (var i = 0; i < layer.length; i++) {
    var feature = layer.feature(i);

    // only consider polygon features with Tiger name or OSM highway tag
    if (feature.type === 2 && (feature.properties.FULLNAME !== '' || feature.properties.highway)) {
      var geom = feature.loadGeometry();

      // normalize line geometry
      var line = [];
      for (var k = 0; k < geom.length; k++) {
        for (var j = 0; j < geom[k].length; j++) {
          line.push([
            geom[k][j].x * 4096 / layer.extent,
            geom[k][j].y * 4096 / layer.extent
          ]);
        }
      }

      // only consider lines that are longer than 20 pixels
      if (dist(line) >= 20) {
        lineclip(line, bbox, lines); // clip to tile bbox
      }
    }
  }
  return lines;
}

function dist(line) { // approximate distance
  var d = 0;
  for (var i = 1; i < line.length; i++) {
    var dx = line[i][0] - line[i - 1][0];
    var dy = line[i][1] - line[i - 1][1];
    d += Math.sqrt(dx * dx + dy * dy);
  }
  return d;
}
<commit_msg>improve the diff example<commit_after>'use strict';

var linematch = require('linematch');
var lineclip = require('lineclip');

module.exports = function(data, tile, done) {

  // filter and normalize input geometry
  var tiger = toLines(data.tiger.tiger2015);
  var streets = toLines(data.osm.osm);

  // find tiger parts that are not covered by streets within 10 pixels;
  // filter out chunks that are too short
  var diff = linematch(tiger, streets, 10).filter(filterShort);

  if (diff.length) {
    // write a feature with the diff as MultiLineString
    process.stdout.write(',' + JSON.stringify({
      type: 'Feature',
      properties: {},
      geometry: {
        type: 'MultiLineString',
        coordinates: toGeoJSON(diff, tile)
      }
    }));
  }

  done(null, null);
};

function toGeoJSON(diff, tile) {
  var size = 4096 * Math.pow(2, tile[2]);
  var x0 = 4096 * tile[0];
  var y0 = 4096 * tile[1];

  for (var i = 0; i < diff.length; i++) {
    for (var j = 0; j < diff[i].length; j++) {
      var p = diff[i][j];
      var y2 = 180 - (p[1] + y0) * 360 / size;
      diff[i][j] = [
        round((p[0] + x0) * 360 / size - 180),
        round(360 / Math.PI * Math.atan(Math.exp(y2 * Math.PI / 180)) - 90)];
    }
  }
  return diff;
}

function round(num) {
  return Math.round(num * 1e6) / 1e6;
}

function toLines(layer) {
  var lines = [];
  var bbox = [0, 0, 4096, 4096];

  for (var i = 0; i < layer.length; i++) {
    var feature = layer.feature(i);

    // only consider polygon features with Tiger name or OSM highway tag
    if (feature.type === 2 && (feature.properties.FULLNAME !== '' || feature.properties.highway)) {
      var geom = feature.loadGeometry();

      for (var k = 0; k < geom.length; k++) {
        lineclip(normalizeLine(geom[k], layer.extent), bbox, lines); // clip to tile bbox and add to result
      }
    }
  }
  return lines;
}

function normalizeLine(line, extent) {
  var newLine = [];
  for (var i = 0; i < line.length; i++) {
    newLine.push([
      line[i].x * 4096 / extent,
      line[i].y * 4096 / extent]);
  }
  return newLine;
}

function filterShort(line) {
  return dist(line) >= 30; // line length is at least 30 pixels
}

function dist(line) { // approximate distance
  var d = 0;
  for (var i = 1; i < line.length; i++) {
    var dx = line[i][0] - line[i - 1][0];
    var dy = line[i][1] - line[i - 1][1];
    d += Math.sqrt(dx * dx + dy * dy);
  }
  return d;
}

{'fraction_non_alphanumeric': 0.1641509433962264, 'fraction_numerical': 0.03815513626834382, 'mean_word_length': 2.4299065420560746, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7280954', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1675, 'n_words': 494}