<commit_before>const fetch = require('../utils/fetch')
const Rx = require('rxjs/Rx')
const config = require('config')
const Loki = require('lokijs')

// /
// /  Get and poll of advisories
// /
const get = fetch(process.env.NSP_URL || config.get('nsp.advisories')).filter(x => x.statusCode === 200).map(x => JSON.parse(x.body))

const poll = get.merge(Rx.Observable.interval(config.get('nsp.refreshInterval')).switchMapTo(get))

const pollWithdistinctUntilChanged = () => {
  const db = new Loki('nsp.json')
  const adv = db.addCollection('advisories')
  return poll.flatMap(x => Rx.Observable.from(x.results)).filter(a => adv.findOne({id: a.id}) === null).do(a => adv.insert(a))
}

module.exports = {poll, get, pollWithdistinctUntilChanged}<commit_msg>fix(nsp): added simple logging stmt for polling from nsp<commit_after>const fetch = require('../utils/fetch')
const Rx = require('rxjs/Rx')
const config = require('config')
const Loki = require('lokijs')

// /
// /  Get and poll of advisories
// /
const get = fetch(process.env.NSP_URL || config.get('nsp.advisories')).filter(x => x.statusCode === 200).map(x => JSON.parse(x.body))

const poll = get.merge(Rx.Observable.interval(config.get('nsp.refreshInterval')).switchMapTo(get))

const pollWithdistinctUntilChanged = () => {
  const db = new Loki('nsp.json')
  const adv = db.addCollection('advisories')
  return poll.do(_ => console.log(`[${new Date().toISOString()}] checked for new advisories`)).flatMap(x => Rx.Observable.from(x.results)).filter(a => adv.findOne({id: a.id}) === null).do(a => adv.insert(a))
}

module.exports = {poll, get, pollWithdistinctUntilChanged}
{'fraction_non_alphanumeric': 0.1898034398034398, 'fraction_numerical': 0.0036855036855036856, 'mean_word_length': 5.011070110701107, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4974431', 'n_tokens_mistral': 564, 'n_tokens_neox': 555, 'n_words': 107}