<commit_before>const dbCmds = require('../db/commands.js')
const moment = require('moment-timezone')
const defaultConfigs = require('../../util/checkConfig.js').defaultConfigs
const log = require('../../util/logger.js')
const Article = require('../../models/Article.js')
const ArticleIDResolver = require('../../structs/ArticleIDResolver.js')

function formatCallbackArticle (article, source, rssName, callbackFunc) {
  // For ArticleMessage to access once ScheduleManager receives this article
  article._delivery = {
    guildId: source.guildId, // Originally set in source through FeedSchedule.js
    dateSettings: source.dateSettings, // Not the actual date, but settings concerning date format/language/timezone. Originally set in source through FeedSchedule.js
    rssName,
    channelId: source.channel,
    source
  }

  callbackFunc({ status: 'article', article })
}

module.exports = async (data, callbackArticle) => {
  const { rssList, articleList, debugFeeds, link, shardId, config, feedData, scheduleName, runNum, useIdType } = data // feedData is only defined when config.database.uri is set to a databaseless folder path
  if (!scheduleName) throw new Error('Missing schedule name for shared logic')
  const totalArticles = articleList.length
  const dbIds = new Set()
  const dbTitles = new Set()
  const dbCustomComparisons = {} // Object with comparison names as key, and array as values whose function is similar to how dbIds and dbTitles work
  const dbCustomComparisonsValid = {} // Object with comparison names as key, and only boolean true values as values
  const dbCustomComparisonsToDelete = []
  const customComparisonsToUpdate = []
  const toInsert = []
  const toUpdate = {} // Article's resolved IDs as key and the article as value
  const collectionID = Article.getCollectionID(link, shardId, scheduleName)
  const Feed = Article.modelByID(collectionID)
  const feedCollectionId = feedData ? collectionID : undefined
  const feedCollection = feedData ? (feedData[feedCollectionId] || []) : undefined

  const docs = await dbCmds.findAll(feedCollection || Feed)
  for (const doc of docs) {
    // Push the main data for built in comparisons
    dbIds.add(doc.id)
    dbTitles.add(doc.title)

    // Now deal with custom comparisons
    const docCustomComparisons = doc.customComparisons
    if (docCustomComparisons !== undefined && Object.keys(docCustomComparisons).length > 0) {
      for (const n in docCustomComparisons) { // n = customComparison's name (such as description, author, etc.)
        if (!dbCustomComparisons[n]) {
          dbCustomComparisons[n] = [docCustomComparisons[n]]
        } else {
          dbCustomComparisons[n].push(docCustomComparisons[n])
        }
      }
    }
  }

  const checkCustomComparisons = Object.keys(dbCustomComparisons).length > 0
  for (const article of articleList) {
    article._id = ArticleIDResolver.getIDTypeValue(article, useIdType)
    if (checkCustomComparisons) {
    // Iterate over the values stored in the db, and see if the custom comparison names in the db exist in any of the articles. If they do, then it is marked valid
      for (const compName in dbCustomComparisons) {
        if (article[compName] !== undefined && (typeof article[compName] !== 'object' || article[compName] === null)) {
          dbCustomComparisonsValid[compName] = true
        }
      }
    }
    if (!dbIds.has(article._id)) {
      toInsert.push(article)
    }
  }

  // If any invalid custom comparisons are found, delete them
  if (checkCustomComparisons) {
    for (const q in dbCustomComparisons) {
      if (!dbCustomComparisonsValid[q]) {
        dbCustomComparisonsToDelete.push(q)
        delete dbCustomComparisons[q]
      }
    }
  }
  await dbCmds.bulkInsert(feedCollection || Feed, toInsert)
  if (dbIds.size === 0) {
    return { status: 'success', link, feedCollection, feedCollectionId }
  }
  for (const rssName in rssList) {
    const source = rssList[rssName]
    const toDebug = debugFeeds && debugFeeds.includes(rssName)
    const customComparisons = source.customComparisons // Array of names
    const sentTitles = new Set()

    if (Array.isArray(customComparisons)) {
      for (let n = customComparisons.length - 1; n >= 0; --n) {
        const name = customComparisons[n]
        if (name === 'title' || name === 'guid' || name === 'pubdate') { // Forbidden custom comparisons since these are already used by the bot
          customComparisons.splice(n, 1)
          continue
        }
        if (!dbCustomComparisons[name] && !customComparisonsToUpdate.includes(name)) {
          customComparisonsToUpdate.push(name) // Since this custom comparison wasn't found in the db, it might be uninitialized or not found in any articles (as checked previously)
        }
      }
    }

    if (toDebug) {
      log.debug.info(`${rssName}: Processing collection. Total article list length: ${totalArticles}`)
    }

    const maxAge = config.feeds.cycleMaxAge
    const cutoffDay = moment().subtract(maxAge, 'days')

    const globalDateCheck = config.feeds.checkDates != null ? config.feeds.checkDates : defaultConfigs.feeds.checkDates.default
    const localDateCheck = source.checkDates
    const checkDate = typeof localDateCheck !== 'boolean' ? globalDateCheck : localDateCheck

    const globalTitleCheck = config.feeds.checkTitles != null ? config.feeds.checkTitles : defaultConfigs.feeds.checkTitles.default
    const localTitleCheck = source.checkTitles
    const checkTitle = typeof globalTitleCheck !== 'boolean' ? globalTitleCheck : localTitleCheck

    if (toDebug) {
      log.debug.info('Database IDs:', JSON.stringify(Array.from(dbIds)))
      log.debug.info('Database Titles:', JSON.stringify(Array.from(dbTitles)))
    }
    for (let a = totalArticles - 1; a >= 0; --a) { // Loop from oldest to newest so the queue that sends articleMessages work properly, sending the older ones first
      const article = articleList[a]
      const notInitialized = dbIds.size === 0 && totalArticles !== 1
      const matchedID = dbIds.has(article._id)
      const matchedTitle = checkTitle && (dbTitles.has(article.title) || sentTitles.has(article.title))
      const matchedDate = checkDate && ((!article.pubdate || article.pubdate.toString() === 'Invalid Date') || (article.pubdate && article.pubdate.toString() !== 'Invalid Date' && article.pubdate < cutoffDay))
      let seen = false
      if (notInitialized || matchedID || matchedTitle || matchedDate) {
        if (toDebug) log.debug.info(`${rssName}: Not sending article (ID: ${article._id}, TITLE: ${article.title}) Matched ${notInitialized ? 'init case' : matchedID ? 'ID' : matchedTitle ? 'title' : matchedDate ? 'date' : 'UNKNOWN CASE'}.`)
        seen = true
      } else if (checkTitle && article.title) {
        sentTitles.add(article.title)
      }

      if (runNum === 0 && config.feeds.sendOldOnFirstCycle === false) {
        if (debugFeeds && debugFeeds.includes(rssName)) {
          log.debug.warning(`${rssName}: Not sending article (ID: ${article._id}, TITLE: ${article.title}), config.feeds.sendOldOnFirstCycle is false`)
        }
        continue
      }

      // Check for extra user-specified comparisons
      if (!seen) {
        formatCallbackArticle(article, source, rssName, callbackArticle)
        continue
      }
      if (!Array.isArray(customComparisons)) {
        continue
      }
      for (const comparisonName of customComparisons) {<commit_msg>Separate module into more functions for shared logic<commit_after>const dbCmds = require('../db/commands.js')
const moment = require('moment-timezone')
const defaultConfigs = require('../../util/checkConfig.js').defaultConfigs
const log = require('../../util/logger.js')
const Article = require('../../models/Article.js')
const ArticleIDResolver = require('../../structs/ArticleIDResolver.js')

/**
 * @param {object} article
 * @param {object} source
 * @param {string} rssName
 * @param {function} callbackFunc
 */
function formatCallbackArticle (article, source, rssName, callbackFunc) {
  // For ArticleMessage to access once ScheduleManager receives this article
  article._delivery = {
    guildId: source.guildId, // Originally set in source through FeedSchedule.js
    dateSettings: source.dateSettings, // Not the actual date, but settings concerning date format/language/timezone. Originally set in source through FeedSchedule.js
    rssName,
    channelId: source.channel,
    source
  }

  callbackFunc({ status: 'article', article })
}

/**
 * @typedef {object} DatabaseData
 * @property {Set<string>} dbIds - Set of stored database IDs
 * @property {Set<string>} dbTitles - Set of stored database titles
 * @property {object<string, string[]>} dbCustomComparisons - Comparison names as keys, and their relevant article values in arrays
 */

/**
 * @param {import('mongoose').Model|object[]} collection
 * @returns {DatabaseData}
 */
async function getDataFromDocuments (collection) {
  const dbIds = new Set()
  const dbTitles = new Set()
  /** @type {object<string, string[]} */
  const dbCustomComparisons = {}
  const docs = await dbCmds.findAll(collection)
  for (const doc of docs) {
    // Push the main data for built in comparisons
    dbIds.add(doc.id)
    dbTitles.add(doc.title)

    // Now deal with custom comparisons
    const docCustomComparisons = doc.customComparisons
    if (docCustomComparisons !== undefined && Object.keys(docCustomComparisons).length > 0) {
      for (const articleProperty in docCustomComparisons) { // n = customComparison's name (such as description, author, etc.)
        const value = docCustomComparisons[articleProperty]
        if (!dbCustomComparisons[articleProperty]) {
          dbCustomComparisons[articleProperty] = [value]
        } else {
          dbCustomComparisons[articleProperty].push(value)
        }
      }
    }
  }

  return { dbIds, dbTitles, dbCustomComparisons }
}

/**
 * @param {object[]} collection
 * @param {Set<string>} dbIds
 * @param {string} useIdType
 * @param {object[]} articleList
 * @param {object<string, string[]>} dbCustomComparisons
 * @returns {Set<string>}
 */
async function articleListTasks (collection, dbIds, useIdType, articleList, dbCustomComparisons) {
  const toInsert = []
  const dbCustomComparisonsValid = new Set()
  const dbCustomComparisonsToDelete = new Set()

  const checkCustomComparisons = Object.keys(dbCustomComparisons).length > 0
  for (const article of articleList) {
    article._id = ArticleIDResolver.getIDTypeValue(article, useIdType)
    if (checkCustomComparisons) {
    // Iterate over the values stored in the db, and see if the custom comparison names in the db exist in any of the articles. If they do, then it is marked valid
      for (const compName in dbCustomComparisons) {
        if (article[compName] !== undefined && (typeof article[compName] !== 'object' || article[compName] === null)) {
          dbCustomComparisonsValid.add(compName)
        }
      }
    }
    if (!dbIds.has(article._id)) {
      toInsert.push(article)
    }
  }

  for (const articleProperty in dbCustomComparisons) {
    if (!dbCustomComparisonsValid.has(articleProperty)) {
      dbCustomComparisonsToDelete.add(articleProperty)
      delete dbCustomComparisons[articleProperty]
    }
  }

  await dbCmds.bulkInsert(collection, toInsert)

  return dbCustomComparisonsToDelete
}

/**
 * @param {object} config - The default config.json
 * @param {object} source - User's source config
 */
function determineArticleChecks (config, source) {
  const globalDateCheck = config.feeds.checkDates != null ? config.feeds.checkDates : defaultConfigs.feeds.checkDates.default
  const localDateCheck = source.checkDates
  const checkDate = typeof localDateCheck !== 'boolean' ? globalDateCheck : localDateCheck

  const globalTitleCheck = config.feeds.checkTitles != null ? config.feeds.checkTitles : defaultConfigs.feeds.checkTitles.default
  const localTitleCheck = source.checkTitles
  const checkTitle = typeof globalTitleCheck !== 'boolean' ? globalTitleCheck : localTitleCheck

  return { checkDate, checkTitle }
}

module.exports = async (data, callbackArticle) => {
  const { rssList, articleList, debugFeeds, link, shardId, config, feedData, scheduleName, runNum, useIdType } = data // feedData is only defined when config.database.uri is set to a databaseless folder path
  if (!scheduleName) throw new Error('Missing schedule name for shared logic')
  const totalArticles = articleList.length
  const customComparisonsToUpdate = []
  const toUpdate = {} // Article's resolved IDs as key and the article as value
  const collectionID = Article.getCollectionID(link, shardId, scheduleName)
  const Feed = Article.modelByID(collectionID)
  const feedCollectionId = feedData ? collectionID : undefined
  const feedCollection = feedData ? (feedData[feedCollectionId] || []) : undefined

  const { dbIds, dbTitles, dbCustomComparisons } = await getDataFromDocuments(feedCollection || Feed)
  const dbCustomComparisonsToDelete = await articleListTasks(dbIds, useIdType, articleList, dbCustomComparisons)

  if (dbIds.size === 0) {
    // Tthe database collection has not been initialized. If a feed has 100 articles, skip everything past this point so it doesn't send 100 articles.
    return { status: 'success', link, feedCollection, feedCollectionId }
  }

  for (const rssName in rssList) {
    const source = rssList[rssName]
    const toDebug = debugFeeds && debugFeeds.includes(rssName)
    const customComparisons = source.customComparisons // Array of names
    const sentTitles = new Set()

    if (Array.isArray(customComparisons)) {
      for (let n = customComparisons.length - 1; n >= 0; --n) {
        const name = customComparisons[n]
        if (name === 'title' || name === 'guid' || name === 'pubdate') { // Forbidden custom comparisons since these are already used by the bot
          customComparisons.splice(n, 1)
          continue
        }
        if (!dbCustomComparisons[name] && !dbCustomComparisonsToDelete.has(name) && !customComparisonsToUpdate.includes(name)) {
          customComparisonsToUpdate.push(name) // Since this custom comparison wasn't found in the db, it might be uninitialized or not found in any articles (as checked previously)
        }
      }
    }

    if (toDebug) {
      log.debug.info(`${rssName}: Processing collection. Total article list length: ${totalArticles}`)
    }

    const maxAge = config.feeds.cycleMaxAge
    const cutoffDay = moment().subtract(maxAge, 'days')

    const { checkDate, checkTitle } = determineArticleChecks

    if (toDebug) {
      log.debug.info('Database IDs:', JSON.stringify(Array.from(dbIds)))
      log.debug.info('Database Titles:', JSON.stringify(Array.from(dbTitles)))
    }
    for (let a = totalArticles - 1; a >= 0; --a) { // Loop from oldest to newest so the queue that sends articleMessages work properly, sending the older ones first
      const article = articleList[a]
      const matchedID = dbIds.has(article._id)
      const matchedTitle = checkTitle && (dbTitles.has(article.title) || sentTitles.has(article.title))
      const matchedDate = checkDate && ((!article.pubdate || article.pubdate.toString() === 'Invalid Date') || (article.pubdate && article.pubdate.toString() !== 'Invalid Date' && article.pubdate < cutoffDay))
      let seen = false
      if (matchedID || matchedTitle || matchedDate) {
        if (toDebug) log.debug.info(`${rssName}: Not sending article (ID: ${article._id}, TITLE: ${article.title}) Matched ${matchedID ? 'ID' : matchedTitle ? 'title' : matchedDate ? 'date' : 'UNKNOWN CASE'}.`)
        seen = true
      } else if (checkTitle && article.title) {
        sentTitles.add(article.title)
      }

      if (runNum === 0 && config.feeds.sendOldOnFirstCycle === false) {
        if (debugFeeds && debugFeeds.includes(rssName)) {
          log.debug.warning(`${rssName}: Not sending article (ID: ${article._id}, TITLE: ${article.title}), config.feeds.sendOldOnFirstCycle is false`)
        }
        continue
      }

      // Check for extra user-specified comparisons
      if (!seen) {
        formatCallbackArticle(article, source, rssName, callbackArticle)
        continue
      }
      if (!Array.isArray(customComparisons)) {
        continue
      }
      for (const comparisonName of customComparisons) {
{'fraction_non_alphanumeric': 0.1112677783227072, 'fraction_numerical': 0.0015939185875429131, 'mean_word_length': 3.651554034787568, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5684990', 'n_tokens_mistral': 4675, 'n_tokens_neox': 4553, 'n_words': 1333}