<commit_before>const _ = require('lodash');
const google = require('googleapis');
const util = require('../util/seeds');
const logger = require('../util/logger')(__filename);
const {knex} = require('../util/database').connect();
const BPromise = require('bluebird');
const requireEnvs = require('../util/require-envs');
const sheets = google.sheets('v4');

requireEnvs([
  'GSHEETS_EVENTS',
  'GSHEETS_API_KEY',
]);

const eventSheets = JSON.parse(process.env.GSHEETS_EVENTS);
const batchGetAsync = BPromise.promisify(sheets.spreadsheets.values.batchGet);
const cities = {};

getEvents();

function getEvents() {
  logger.info("Updating events data");
  return knex('cities').select('*').then(rows => {
    rows.forEach(city => {
      cities[city.name] = city.id;
    });
  })
  .then(() => BPromise.map(eventSheets, eventSheet => {
    const request = {
      spreadsheetId: eventSheet.id,
      ranges: ['A:M'],
      valueRenderOption: 'UNFORMATTED_VALUE',
      key: process.env.GSHEETS_API_KEY,
    };

    return batchGetAsync(request).then(response => {
      let payload = _.get(response, 'valueRanges[0].values', null);

      if (!payload || payload.length === 0) {
        BPromise.resolve();
      }

      return _extractAndSaveData(payload, eventSheet);
    });
  }))
  .then(() =>{
    logger.info("Event data updated");
  })
  .catch(err => {
    logger.error("Could not successfully complete event update", err);
  });
}
<commit_msg>Exit the process when done<commit_after>const _ = require('lodash');
const google = require('googleapis');
const util = require('../util/seeds');
const logger = require('../util/logger')(__filename);
const {knex} = require('../util/database').connect();
const BPromise = require('bluebird');
const requireEnvs = require('../util/require-envs');
const sheets = google.sheets('v4');

requireEnvs([
  'GSHEETS_EVENTS',
  'GSHEETS_API_KEY',
]);

const eventSheets = JSON.parse(process.env.GSHEETS_EVENTS);
const batchGetAsync = BPromise.promisify(sheets.spreadsheets.values.batchGet);
const cities = {};

getEvents()
.then(() =>{
  logger.info("Event data updated");
  process.exit();
})
.catch(err => {
  logger.error("Could not successfully complete event update", err);
  process.exit();
});

function getEvents() {
  logger.info("Updating events data");
  return knex('cities').select('*').then(rows => {
    rows.forEach(city => {
      cities[city.name] = city.id;
    });
  })
  .then(() => BPromise.map(eventSheets, eventSheet => {
    const request = {
      spreadsheetId: eventSheet.id,
      ranges: ['A:M'],
      valueRenderOption: 'UNFORMATTED_VALUE',
      key: process.env.GSHEETS_API_KEY,
    };

    return batchGetAsync(request).then(response => {
      let payload = _.get(response, 'valueRanges[0].values', null);

      if (!payload || payload.length === 0) {
        BPromise.resolve();
      }

      return _extractAndSaveData(payload, eventSheet);
    });
  }))
}

{'fraction_non_alphanumeric': 0.17801225323349218, 'fraction_numerical': 0.0020422055820285907, 'mean_word_length': 3.939495798319328, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3681909', 'n_tokens_mistral': 957, 'n_tokens_neox': 946, 'n_words': 176}