<commit_before>/**
 * This is the main code to kick off the data fetching processes, handle their
 * results, saving to a database and repeating the process... forever.
 *
 * There are helpful command line shortcuts, all described with
 * `node index.js --help`.
 */
'use strict';

// Set up command line arguments
var argv = require('yargs')
  .usage('Usage: $0 --dryrun --source \'Beijing US Embassy\'')
  .boolean('dryrun')
  .describe('dryrun', 'Run the fetch process but do not attempt to save to the database and instead print to console, useful for testing.')
  .alias('d', 'dryrun')
  .describe('source', 'Run the fetch process with only the defined source using source name.')
  .alias('s', 'source')
  .nargs('source', 1)
  .help('h')
  .alias('h', 'help')
  .argv;

var async = require('async');
import { assign, filter, pick, chain, find } from 'lodash';
var knex = require('knex');
let knexConfig = require('./knexfile');
var utils = require('./lib/utils');
var request = require('request');
var log = require('./lib/logger');

var adapters = require('./adapters');
var sources = require('./sources');

var apiURL = process.env.API_URL || 'http://localhost:3004/v1/webhooks';
var webhookKey = process.env.WEBHOOK_KEY || '123';
let pg;
let st;

// Flatten the sources into a single array, taking into account sources argument<commit_msg>Add a timeout to overall fetch process<commit_after>/**
 * This is the main code to kick off the data fetching processes, handle their
 * results, saving to a database and repeating the process... forever.
 *
 * There are helpful command line shortcuts, all described with
 * `node index.js --help`.
 */
'use strict';

// Set up command line arguments
var argv = require('yargs')
  .usage('Usage: $0 --dryrun --source \'Beijing US Embassy\'')
  .boolean('dryrun')
  .describe('dryrun', 'Run the fetch process but do not attempt to save to the database and instead print to console, useful for testing.')
  .alias('d', 'dryrun')
  .describe('source', 'Run the fetch process with only the defined source using source name.')
  .alias('s', 'source')
  .nargs('source', 1)
  .help('h')
  .alias('h', 'help')
  .argv;

var async = require('async');
import { assign, filter, pick, chain, find } from 'lodash';
var knex = require('knex');
let knexConfig = require('./knexfile');
var utils = require('./lib/utils');
var request = require('request');
var log = require('./lib/logger');

var adapters = require('./adapters');
var sources = require('./sources');

const apiURL = process.env.API_URL || 'http://localhost:3004/v1/webhooks';  // The url to ping on completion
const webhookKey = process.env.WEBHOOK_KEY || '123';  // Secret key to auth with API
const processTimeout = process.env.PROCESS_TIMEOUT || 10 * 60 * 1000; // Kill the process after a certain time in case it hangs
let pg;
let st;

// This is a top-level safety mechanism, we'll kill this process after a certain
// time in case it's hanging. Intended to avoid https://github.com/openaq/openaq-fetch/issues/154
setTimeout(() => {
  log.error('Uh oh, process timed out.');
  process.exit(1);
}, processTimeout);

// Flatten the sources into a single array, taking into account sources argument
{'fraction_non_alphanumeric': 0.14682911590128084, 'fraction_numerical': 0.00999687597625742, 'mean_word_length': 4.0664556962025316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6968852', 'n_tokens_mistral': 985, 'n_tokens_neox': 934, 'n_words': 376}