<commit_before>var collect = require('collect-stream')
var pumpify = require('pumpify')
var mapStream = require('through2-map')
var collectTransform = require('collect-transform-stream')
var defork = require('osm-p2p-defork')

var refs2nodes = require('../lib/util').refs2nodes
var checkRefExist = require('../lib/check_ref_ex.js')

module.exports = function (osm) {
  return function getMap (bbox, opts, cb) {
    if (typeof opts === 'function') {
      cb = opts
      opts = {}
    }
    opts.forks = opts.forks || false

    var query = [[bbox[1], bbox[3]], [bbox[0], bbox[2]]] // left,bottom,right,top
    var pipeline = [
      osm.queryStream(query, opts),
      checkRefExist(osm),
      mapStream.obj(refs2nodes)
    ]<commit_msg>fix: filter deletions coming from osm-p2p-db<commit_after>var collect = require('collect-stream')
var pumpify = require('pumpify')
var mapStream = require('through2-map')
var collectTransform = require('collect-transform-stream')
var defork = require('osm-p2p-defork')
var through = require('through2')

var refs2nodes = require('../lib/util').refs2nodes
var checkRefExist = require('../lib/check_ref_ex.js')

module.exports = function (osm) {
  return function getMap (bbox, opts, cb) {
    if (typeof opts === 'function') {
      cb = opts
      opts = {}
    }
    opts.forks = opts.forks || false

    var query = [[bbox[1], bbox[3]], [bbox[0], bbox[2]]] // left,bottom,right,top

    // For now, filter deletions (until downstream can handle them)
    var deletionFilter = through.obj(function (chunk, enc, next) {
      if (!chunk.deleted) next(null, chunk)
      else next()
    })

    var pipeline = [
      osm.queryStream(query, opts),
      deletionFilter,
      checkRefExist(osm),
      mapStream.obj(refs2nodes)
    ]
{'fraction_non_alphanumeric': 0.15302089215132694, 'fraction_numerical': 0.01129305477131564, 'mean_word_length': 3.5670103092783507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5472254', 'n_tokens_mistral': 601, 'n_tokens_neox': 583, 'n_words': 136}