<commit_before>var app = require('../scanner.js');
var fs = require('fs');
var lazy = require('lazy');
var async = require('async');
var MongoClient = require('mongodb').MongoClient;
var mongodb = require('mongodb');

var scanner = app.createScanner({
  charset: 'UTF-8',
  scanInterval: 30,
  concurrent: 5
});

var sliceStart = 0;
var sliceEnd = 10;

var feedList = [];
var time;

var numfired = 0;
var numerrors = 0;

var entries;

// Create the processFeed function
var processFeed = function (err, feed, result, callback) {
  if (err) {
    numerrors += 1;
    console.log('%s : %s', feed, err);
    // Return callback on error
    return callback(err);
  };
  var diff = process.hrtime(time);
  numfired += 1;
  if (result.meta) {
    console.log('%ds:%dms, Feed: %s, Title: %s', diff[0],diff[1] / 1000000, feed, result.meta.title);
  } else {
    console.log('no meta for %s', result.feed);
  };
  // Add articles to database
  var bulkop = entries.initializeUnorderedBulkOp();

  result.articles.forEach(function (item, index, array) {
    bulkop.find({id: item.guid}).upsert().updateOne(item);
  });
  
  bulkop.execute(function (err, result) {
    if (err) return callback(err);
    return callback();
  });
};

async.series({
  // Initialise the database
  db: function (callback) {
    MongoClient.connect('mongodb://localhost:27017/feedscanner', function (err, db) {
      if (err) return callback(err);
      entries = new mongodb.Collection(db, 'entries');
      console.log('Connected to database');
      callback(null);
    });
  },
  // Read the feeds from a file
  lazy: function (callback) {
    new lazy(fs.createReadStream('../test/superfeedr_popular_feeds.txt'))
    .on('end', function () {
      console.log('Loaded feeds');
      callback(null);
    })
    .lines<commit_msg>Added check for empty bulkop<commit_after>var app = require('../scanner.js');
var fs = require('fs');
var lazy = require('lazy');
var async = require('async');
var MongoClient = require('mongodb').MongoClient;
var mongodb = require('mongodb');

var scanner = app.createScanner({
  charset: 'UTF-8',
  scanInterval: 30,
  concurrent: 5
});

var sliceStart = 0;
var sliceEnd = 10;

var feedList = [];
var time;

var numfired = 0;
var numerrors = 0;

var entries;

// Create the processFeed function
var processFeed = function (err, feed, result, callback) {
  if (err) {
    numerrors += 1;
    console.log('%s : %s', feed, err);
    // Return callback on error
    return callback(err);
  };
  var diff = process.hrtime(time);
  numfired += 1;
  if (result.meta) {
    console.log('%ds:%dms, Feed: %s, Title: %s', diff[0],diff[1] / 1000000, feed, result.meta.title);
  } else {
    console.log('no meta for %s', result.feed);
  };
  // Add articles to database
  var bulkop = entries.initializeUnorderedBulkOp();
  var ops = 0;

  result.articles.forEach(function (item, index, array) {
    bulkop.find({id: item.guid}).upsert().updateOne(item);
    ops += 1;
  });
  
  if (ops >= 1) {
    bulkop.execute(function (err, result) {
      if (err) return callback(err);
      return callback();
    });
  } else {
    return callback();
  };
};

async.series({
  // Initialise the database
  db: function (callback) {
    MongoClient.connect('mongodb://localhost:27017/feedscanner', function (err, db) {
      if (err) return callback(err);
      entries = new mongodb.Collection(db, 'entries');
      console.log('Connected to database');
      callback(null);
    });
  },
  // Read the feeds from a file
  lazy: function (callback) {
    new lazy(fs.createReadStream('../test/superfeedr_popular_feeds.txt'))
    .on('end', function () {
      console.log('Loaded feeds');
      callback(null);
    })
    .lines
{'fraction_non_alphanumeric': 0.16401831403178024, 'fraction_numerical': 0.014274171828709938, 'mean_word_length': 3.395266272189349, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2527414', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1218, 'n_words': 322}