<commit_before># osm-p2p-db

p2p database for open street map data

# example

``` js
var hyperlog = require('hyperlog')

var level = require('level')
var db = {
  log: level('/tmp/osm-p2p/log'),
  index: level('/tmp/osm-p2p/index')
}
var fdstore = require('fd-chunk-store')
var storefile = '/tmp/osm-p2p/kdb'

var osmdb = require('osm-p2p-db')
var osm = osmdb({
  log: hyperlog(db.log, { valueEncoding: 'json' }),
  db: db.index,
  store: fdstore(4096, storefile),
  size: 4096
})

if (process.argv[2] === 'create') {
  var value = JSON.parse(process.argv[3])
  osm.create(value, function (err, key, node) {
    if (err) console.error(err)
    else console.log(key)
  })
} else if (process.argv[2] === 'query') {
  var q = process.argv.slice(3).map(csplit)
  osm.query(q, function (err, pts) {
    if (err) console.error(err)
    else pts.forEach(function (pt) {
      console.log(pt)
    })
  })
}

function csplit (x) { return x.split(',').map(Number) }
```

Now we can create a few nodes and search with a bounding box query:

```
$ mkdir /tmp/osm-p2p
$ node db.js create '{"type":"node","lat":64.6,"lon":-147.8}'
a50aa575ae96971b
$ node db.js create '{"type":"node","lat":64.3,"lon":-148.2}'
1b83545b2b06eaad
$ node db.js create '{"type":"node","lat":64.5,"lon":-147.3}'
c328c306ddcce256
$ node db.js query 64.1,64.6 -148,-147
{ type: 'node',
  lat: 64.5,
  lon: -147.3,
  id: 'c328c306ddcce256',
  version: 'e635d07b9fc0a9d048cdd5d9e97a44a19ba3a0b2a51830d1e3e0fadcb80935fc' }
```

We can make a `way` document that refers to a list of `node` documents:

```
$ node db.js create '{"type":"way","refs":
["a50aa575ae96971b","1b83545b2b06eaad","c328c306ddcce256"]}'
cb8b6842a9114b76
```

When we query, any `ways` that have one or more nodes within the bounding box
will turn up in the results:

```
$ node db.js query 64.1,64.6 -148,-147
{ type: 'node',
  lat: 64.5,
  lon: -147.3,
  id: 'c328c306ddcce256',
  version: 'e635d07b9fc0a9d048cdd5d9e97a44a19ba3a0b2a51830d1e3e0fadcb80935fc' }
{ type: 'way',
  refs: [ 'a50aa575ae96971b', '1b83545b2b06eaad', 'c328c306ddcce256' ],
  id: 'cb8b6842a9114b76',
  version: 'f4fc0045e298ca4f9373fab78dee4f0561b4056dcd7975eb92f21d0a05e0eede' }
```

# api

``` js
var osmdb = require('osm-p2p-db')
```

## var osm = osmdb(opts)

Create a new `osm` instance with:

* `opts.log` - a [hyperlog][1] with a valueEncoding of `json`
* `opts.db` - a [levelup][2] instance to store index data
* `opts.store` - an [abstract-chunk-store][3] instance
* `opts.size` - the size of the chunks in `opts.store`

You may safely delete the index database whenever you like. The index data is
automatically regenerated. This is very useful if there are breaking changes to
the index code or if the data becomes corrupted. The hyperlog contains the
source of truth.

[1]: https://npmjs.com/package/hyperlog
[2]: https://npmjs.com/package/levelup
[3]: https://npmjs.com/package/abstract-chunk-store

## osm.create(doc, opts={}, cb)

Store a new document from `doc`. `cb(err, id, node)` fires with the generated
OSM `id` and the `node` from the underlying hyperlog.

* Nodes should have `doc.lat` and `doc.lon` coordinates.
* Ways should have an array of OSM keys as `doc.refs`.
* Relations should have an array of OSM keys as `doc.members`.

## osm.put(id, doc, opts={}, cb)

Replace a document at `id` with `doc`.

If the document didn't exist previously, it will be created.

The options `opts` are passed to the underlying [hyperkv][4] instance.

[4]: https://npmjs.com/package/hyperkv

## osm.get(id, opts={}, cb)

Get a document as `cb(err, docs)` by its OSM `id`.

`docs` is an object mapping hyperlog hashes to current document values.

The options `opts` are passed to the underlying [hyperkv][4] instance.

## osm.query(q, opts, cb)

Query for all nodes, ways, and relations in the query given by the array `q`.
Queries are arrays of `[[minLat,maxLat],[minLon,maxLon]]` specifying a bounding
box.

`cb(err, res)` fires with an array of results `res`. Each result is the
document augmented with an `id` property and a `version` property that is the
hash key from the underlying hyperlog.

## var rstream = osm.queryStream(q, opts)

Return a readable object stream `rstream` of query results contained in the
query `q`. Queries are arrays of `[[minLat,maxLat],[minLon,maxLon]]` specifying
a bounding box.

Each object in the stream is a document augmented with an `id` property and a
`version` property that is the hash key from the underlying hyperlog.

# browser<commit_msg>document hyperkv merge behavior with opts.links, fixes issue #8<commit_after># osm-p2p-db

p2p database for open street map data

# example

``` js
var hyperlog = require('hyperlog')

var level = require('level')
var db = {
  log: level('/tmp/osm-p2p/log'),
  index: level('/tmp/osm-p2p/index')
}
var fdstore = require('fd-chunk-store')
var storefile = '/tmp/osm-p2p/kdb'

var osmdb = require('osm-p2p-db')
var osm = osmdb({
  log: hyperlog(db.log, { valueEncoding: 'json' }),
  db: db.index,
  store: fdstore(4096, storefile),
  size: 4096
})

if (process.argv[2] === 'create') {
  var value = JSON.parse(process.argv[3])
  osm.create(value, function (err, key, node) {
    if (err) console.error(err)
    else console.log(key)
  })
} else if (process.argv[2] === 'query') {
  var q = process.argv.slice(3).map(csplit)
  osm.query(q, function (err, pts) {
    if (err) console.error(err)
    else pts.forEach(function (pt) {
      console.log(pt)
    })
  })
}

function csplit (x) { return x.split(',').map(Number) }
```

Now we can create a few nodes and search with a bounding box query:

```
$ mkdir /tmp/osm-p2p
$ node db.js create '{"type":"node","lat":64.6,"lon":-147.8}'
a50aa575ae96971b
$ node db.js create '{"type":"node","lat":64.3,"lon":-148.2}'
1b83545b2b06eaad
$ node db.js create '{"type":"node","lat":64.5,"lon":-147.3}'
c328c306ddcce256
$ node db.js query 64.1,64.6 -148,-147
{ type: 'node',
  lat: 64.5,
  lon: -147.3,
  id: 'c328c306ddcce256',
  version: 'e635d07b9fc0a9d048cdd5d9e97a44a19ba3a0b2a51830d1e3e0fadcb80935fc' }
```

We can make a `way` document that refers to a list of `node` documents:

```
$ node db.js create '{"type":"way","refs":
["a50aa575ae96971b","1b83545b2b06eaad","c328c306ddcce256"]}'
cb8b6842a9114b76
```

When we query, any `ways` that have one or more nodes within the bounding box
will turn up in the results:

```
$ node db.js query 64.1,64.6 -148,-147
{ type: 'node',
  lat: 64.5,
  lon: -147.3,
  id: 'c328c306ddcce256',
  version: 'e635d07b9fc0a9d048cdd5d9e97a44a19ba3a0b2a51830d1e3e0fadcb80935fc' }
{ type: 'way',
  refs: [ 'a50aa575ae96971b', '1b83545b2b06eaad', 'c328c306ddcce256' ],
  id: '<KEY>6',
  version: 'f4fc0045e298ca4f9373fab78dee4f0561b4056dcd7975eb92f21d0a05e0eede' }
```

# api

``` js
var osmdb = require('osm-p2p-db')
```

## var osm = osmdb(opts)

Create a new `osm` instance with:

* `opts.log` - a [hyperlog][1] with a valueEncoding of `json`
* `opts.db` - a [levelup][2] instance to store index data
* `opts.store` - an [abstract-chunk-store][3] instance
* `opts.size` - the size of the chunks in `opts.store`

You may safely delete the index database whenever you like. The index data is
automatically regenerated. This is very useful if there are breaking changes to
the index code or if the data becomes corrupted. The hyperlog contains the
source of truth.

[1]: https://npmjs.com/package/hyperlog
[2]: https://npmjs.com/package/levelup
[3]: https://npmjs.com/package/abstract-chunk-store

## osm.create(doc, opts={}, cb)

Store a new document from `doc`. `cb(err, id, node)` fires with the generated
OSM `id` and the `node` from the underlying hyperlog.

* Nodes should have `doc.lat` and `doc.lon` coordinates.
* Ways should have an array of OSM keys as `doc.refs`.
* Relations should have an array of OSM keys as `doc.members`.

## osm.put(id, doc, opts={}, cb)

Replace a document at `id` with `doc`.

If the document didn't exist previously, it will be created.

The options `opts` are passed to the underlying [hyperkv][4] instance.

By default, hyperkv will merge the most recent known forks into a single fork.
To add modifications to a fork without merging the changes into other forks,
set `opts.links` to an array of only the single key you want to update.

[4]: https://npmjs.com/package/hyperkv

## osm.get(id, opts={}, cb)

Get a document as `cb(err, docs)` by its OSM `id`.

`docs` is an object mapping hyperlog hashes to current document values.

The options `opts` are passed to the underlying [hyperkv][4] instance.

## osm.query(q, opts, cb)

Query for all nodes, ways, and relations in the query given by the array `q`.
Queries are arrays of `[[minLat,maxLat],[minLon,maxLon]]` specifying a bounding
box.

`cb(err, res)` fires with an array of results `res`. Each result is the
document augmented with an `id` property and a `version` property that is the
hash key from the underlying hyperlog.

## var rstream = osm.queryStream(q, opts)

Return a readable object stream `rstream` of query results contained in the
query `q`. Queries are arrays of `[[minLat,maxLat],[minLon,maxLon]]` specifying
a bounding box.

Each object in the stream is a document augmented with an `id` property and a
`version` property that is the hash key from the underlying hyperlog.

# browser
{'fraction_non_alphanumeric': 0.14262595749271767, 'fraction_numerical': 0.06764483763081239, 'mean_word_length': 4.349105597230237, 'pattern_counts': {'":': 22, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2706204', 'n_tokens_mistral': 3808, 'n_tokens_neox': 3416, 'n_words': 1086}