<commit_before>#
# NTB MongoDB updater script
#
# @author <NAME>
#

"use strict"

mongodb = require 'mongodb'
database = require './../database'

database.connect 'ntb_07', (err, db) ->
  return console.log 'db err', err if err


  db.on 'error', (error, db) ->
    console.log 'db.error', error
    db.close()
  
  console.log 'Database connection is open'

  db.collection 'turer', (err, collection) ->
    if err
      console.log 'db connection failed'
      console.log err
      return db.close()
    
    console.log "Collection is open"

    update = (doc, cb) ->
      #if doc.privat.lenker
      #  for link in doc.privat.lenker
      #    if not link.url.match(/^http(s?):\/\//m)
      #      console.log link.url
      #      #return cb new Error('end')
      #
      #if doc.privat.eier
      #  for eier in doc.privat.eier
      #    if eier.url and not eier.url.match(/^http(s?):\/\//m)
      #      console.log eier.url
      
      cb()

    collection.find {"privat" : {$exists: true}}, (err, cursor) ->
      console.log 'collection.find run'

      if err
        console.log 'collection.find() failed'
        console.log err
        return db.close()

      database.each cursor, update, (err, count) ->
        console.log 'count', count

        if err
          console.log 'database update failed'
          console.log err
        else
          console.log 'databse update success'

        db.close()


        <commit_msg>Uses new database connect method<commit_after>#
# NTB MongoDB updater script
#
# @author <NAME>
#

"use strict"

mongodb = require 'mongodb'
database = require './../database.coffee'

database.connect 'ntb_07', (err, db) ->
  return console.log err if err

  console.log 'Database connection is open'

  db.collection 'bilder', (err, collection) ->
    return console.log err if err
    
    console.log "Collection is open"

    cursor = collection.find().limit(2)
    #$where : "this.geojson && this.geojson.coordinates[0] > 50"
    
    database.each cursor, (doc, cb) ->
      console.log doc
      cb()
    , (err, i) ->
      console.log 'cursor end', i
      db.close()

{'fraction_non_alphanumeric': 0.13527477689055895, 'fraction_numerical': 0.0037576326914044154, 'mean_word_length': 2.164933135215453, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5972687', 'n_tokens_mistral': 712, 'n_tokens_neox': 685, 'n_words': 195}