<commit_before>var
  cfg            = require('config'),
  drive          = require('./driveUploader.js'),
  log4js         = require('log4js'),
  reporter       = require('./reporter.js'),
  promisify      = require('util').promisify,
  // This will but set if the configured payslipGetter model doesn't exist
  payslipGetter  = require(`./payslipGetter${cfg.payslipGetter}`);



/*
* Download the monthly payslip and upload it to my Google Drive
*
*/

  /*
  * Initialize
  */


  // logs

  log4js.configure(cfg.log.log4jsConfigs);

  var log = log4js.getLogger(cfg.log.appName);
  log.setLevel(cfg.log.level);


module.exports = async function () {

  /*
   * Main program
   */

  log.info('Begin script');
  log.info('============');

  await main({
    useEmailTrigger: (typeof cfg.useEmailTrigger === undefined)? true   : cfg.useEmailTrigger,
    downloadMode:    (typeof cfg.downloadMode    === undefined)? 'sync' : cfg.downloadMode
  })
}


/**
 * Downloads the one latest payslip in the repo
 * 
 * @returns {Promise<string[]>} A single item array showing where on disk the payslip has been stored
 */
async function downloadLatestPayslip() {
  // Download the payslip
  log.info('Downloading payslip...')
  const downloadedFileLocation = await payslipGetter.downloadPayslip()
  log.info('Payslip downloaded.')
  return [downloadedFileLocation]
}

/**
 * Gets the list of payslips in gdrive, compares them to the
 * payslip provider repo, and downloads ones that are missing.
 * 
 * @returns {Promise<string[]>} Array showing where on disk the payslips have been stored
 */
async function downloadSyncedPayslips() {

  // Get the list of payslips in google drive
  // And the list from the payslip provider
  const [payslipsInDrive, payslipsWithProvider] = await Promise.all([
    drive.getKnownPayslips(),
    payslipGetter.getKnownPayslips()
  ])

  // Look for ones in the payslip provider that are missing in gdrive
  const missingPayslips = payslipsWithProvider
    .filter((payslipWithProvider) => {
      const found = payslipsInDrive.find((p) => {return (p.date === payslipWithProvider.date)})
      return (typeof found === 'undefined')
    })

  log.info('Missing payslips are: ' + JSON.stringify(missingPayslips))
 
  // Download the missing ones
  const downloadedFileLocations = await Promise.all(
    missingPayslips
    .map(async (mp) => {
      log.info(`Downloading payslip ${mp.date}...`)
      const downloadedFileLocation = await payslipGetter.downloadPayslip(Object.assign(mp,{suffix: cfg.companyName}))
      log.info(`Payslip ${mp.date} downloaded.`)
      return downloadedFileLocation
    })
  )
  
  return downloadedFileLocations
}

/**
 * @desc Check if processing is required
 * 
 * @return {Promise<boolean>} isProcessingRequired
 */
async function isProcessingRequired() {

  const triggerChecker = require('./triggerChecker.js');
<commit_msg>Better logging<commit_after>var
  cfg            = require('config'),
  drive          = require('./driveUploader.js'),
  log4js         = require('log4js'),
  reporter       = require('./reporter.js'),
  promisify      = require('util').promisify,
  // This will but set if the configured payslipGetter model doesn't exist
  payslipGetter  = require(`./payslipGetter${cfg.payslipGetter}`);



/*
* Download the monthly payslip and upload it to my Google Drive
*
*/

  /*
  * Initialize
  */


  // logs

  log4js.configure(cfg.log.log4jsConfigs);

  var log = log4js.getLogger(cfg.log.appName);
  log.setLevel(cfg.log.level);


module.exports = async function () {

  /*
   * Main program
   */

  log.info('Begin script');
  log.info('============');

  await main({
    useEmailTrigger: (typeof cfg.useEmailTrigger === undefined)? true   : cfg.useEmailTrigger,
    downloadMode:    (typeof cfg.downloadMode    === undefined)? 'sync' : cfg.downloadMode
  })
}


/**
 * Downloads the one latest payslip in the repo
 * 
 * @returns {Promise<string[]>} A single item array showing where on disk the payslip has been stored
 */
async function downloadLatestPayslip() {
  // Download the payslip
  log.info('Downloading payslip...')
  const downloadedFileLocation = await payslipGetter.downloadPayslip()
  log.info('Payslip downloaded.')
  return [downloadedFileLocation]
}

/**
 * Gets the list of payslips in gdrive, compares them to the
 * payslip provider repo, and downloads ones that are missing.
 * 
 * @returns {Promise<string[]>} Array showing where on disk the payslips have been stored
 */
async function downloadSyncedPayslips() {

  // Get the list of payslips in google drive
  // And the list from the payslip provider
  const [payslipsInDrive, payslipsWithProvider] = await Promise.all([
    drive.getKnownPayslips(),
    payslipGetter.getKnownPayslips()
  ])

  // Look for ones in the payslip provider that are missing in gdrive
  const missingPayslips = payslipsWithProvider
    .filter((payslipWithProvider) => {
      log.debug(`Comparing payslip ${JSON.stringify(payslipWithProvider)}...`)
      const found = payslipsInDrive.find((p) => {
        log.debug(`\tTo ${JSON.stringify(p)}...`)
        return (p.date === payslipWithProvider.date && p.company === cfg.companyName)
      })
      return (typeof found === 'undefined')
    })

  log.info('Missing payslips are: ' + JSON.stringify(missingPayslips))
 
  // Download the missing ones
  const downloadedFileLocations = await Promise.all(
    missingPayslips
    .map(async (mp) => {
      log.info(`Downloading payslip ${mp.date}...`)
      const downloadedFileLocation = await payslipGetter.downloadPayslip(Object.assign(mp,{suffix: cfg.companyName}))
      log.info(`Payslip ${mp.date} downloaded.`)
      return downloadedFileLocation
    })
  )
  
  return downloadedFileLocations
}

/**
 * @desc Check if processing is required
 * 
 * @return {Promise<boolean>} isProcessingRequired
 */
async function isProcessingRequired() {

  const triggerChecker = require('./triggerChecker.js');

{'fraction_non_alphanumeric': 0.12502100487313056, 'fraction_numerical': 0.0016803898504453034, 'mean_word_length': 3.529680365296804, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6158025', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1766, 'n_words': 470}