<commit_before>import assert = require('assert');
import async = require('async');
import util = require('util');
import { MessageQueue } from '../../lib/message_queue';

// TODO: make test assertions beyond lack of error
// TODO: waterfalls
// TODO: put queue size verification in some kind of neat wrapper

function dumpDb(db, callback) {
//  return callback(null, []);
  // TODO: this is out of place; table name is hard-coded, cf CRAWL_TASKS_TABLE_NAME in MessageQueue.js
  const queryStr = 'SELECT *, rowid from crawl_tasks';
  const stmt = db.prepare(queryStr);
  stmt.all((err, rows) => {
    if (err) {
      return callback(err);
    }
    stmt.finalize((err) => {
      callback(err, rows);
    });
  });
}

function verifyQueueSize(queue, expected_size, callback) {
  queue.size((err, result) => {<commit_msg>quick fix for quick fix<commit_after>import assert = require('assert');
import async = require('async');
import util = require('util');
import { MessageQueue } from '../../lib/message_queue';

// TODO: make test assertions beyond lack of error
// TODO: waterfalls
// TODO: put queue size verification in some kind of neat wrapper

function dumpDb(db, callback) {
  return callback(null, []);
  // TODO: this is out of place; table name is hard-coded, cf CRAWL_TASKS_TABLE_NAME in MessageQueue.js
  // const queryStr = 'SELECT *, rowid from crawl_tasks';
  // const stmt = db.prepare(queryStr);
  // stmt.all((err, rows) => {
  //   if (err) {
  //     return callback(err);
  //   }
  //   stmt.finalize((err) => {
  //     callback(err, rows);
  //   });
  // });
}

function verifyQueueSize(queue, expected_size, callback) {
  queue.size((err, result) => {
{'fraction_non_alphanumeric': 0.15137067938021453, 'fraction_numerical': 0.0, 'mean_word_length': 3.5013404825737267, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '119139', 'n_tokens_mistral': 522, 'n_tokens_neox': 514, 'n_words': 158}