<commit_before>const logLinkErrs = require('../config.js').log.linkErrs
const connectDb = require('./db/connect.js')
const processSources = require('./logic/shared.js')
const log = require('../util/logger.js')
const FeedFetcher = require('../util/FeedFetcher.js')

async function getFeed (data, callback) {
  const { link, rssList, uniqueSettings } = data
  try {
    const stream = await FeedFetcher.fetchURL(link, uniqueSettings)
    callback()
    const { articleList, idType } = await FeedFetcher.parseStream(stream, link)
    if (articleList.length === 0) return process.send({ status: 'success', link: link })
    processSources({ articleList, useIdType: idType, ...data }, (err, results) => {
      if (err) log.cycle.error(`Cycle logic`, err, true)
      if (results) process.send(results)
    })
  } catch (err) {
    if (logLinkErrs) log.cycle.warning(`Skipping ${link}`, err)
    process.send({ status: 'failed', link: link, rssList: rssList })
    callback()
  }
}

process.on('message', m => {
  const currentBatch = m.currentBatch
  const config = m.config
  const shardId = m.shardId
  const debugFeeds = m.debugFeeds
  const feedData = m.feedData // Only defined if config.database.uri is set to a databaseless folder path
  const scheduleName = m.scheduleName
  const runNum = m.runNum
  connectDb(true).then(() => {
    const len = Object.keys(currentBatch).length
    let c = 0
    for (var link in currentBatch) {
      const rssList = currentBatch[link]
      let uniqueSettings
      for (var modRssName in rssList) {
        if (rssList[modRssName].advanced && Object.keys(rssList[modRssName].advanced).length > 0) {
          uniqueSettings = rssList[modRssName].advanced
        }
      }
      getFeed({ link, rssList, uniqueSettings, shardId, debugFeeds, config, feedData, scheduleName, runNum }, () => {
        if (++c === len) process.send({ status: 'batch_connected' })
      })
    }
  }).catch(err => log.general.error(`isolatedMethod db connection`, err))
})
<commit_msg>Fix case where isolated calls callback twice<commit_after>const logLinkErrs = require('../config.js').log.linkErrs
const connectDb = require('./db/connect.js')
const processSources = require('./logic/shared.js')
const log = require('../util/logger.js')
const FeedFetcher = require('../util/FeedFetcher.js')

async function getFeed (data, callback) {
  const { link, rssList, uniqueSettings } = data
  let calledbacked = false
  try {
    const stream = await FeedFetcher.fetchURL(link, uniqueSettings)
    callback()
    calledbacked = true
    const { articleList, idType } = await FeedFetcher.parseStream(stream, link)
    if (articleList.length === 0) return process.send({ status: 'success', link: link })
    processSources({ articleList, useIdType: idType, ...data }, (err, results) => {
      if (err) log.cycle.error(`Cycle logic`, err, true)
      if (results) process.send(results)
    })
  } catch (err) {
    if (logLinkErrs) log.cycle.warning(`Skipping ${link}`, err)
    process.send({ status: 'failed', link: link, rssList: rssList })
    if (!calledbacked) {
      callback()
    }
    calledbacked = true
  }
}

process.on('message', m => {
  const currentBatch = m.currentBatch
  const config = m.config
  const shardId = m.shardId
  const debugFeeds = m.debugFeeds
  const feedData = m.feedData // Only defined if config.database.uri is set to a databaseless folder path
  const scheduleName = m.scheduleName
  const runNum = m.runNum
  connectDb(true).then(() => {
    const len = Object.keys(currentBatch).length
    let c = 0
    for (var link in currentBatch) {
      const rssList = currentBatch[link]
      let uniqueSettings
      for (var modRssName in rssList) {
        if (rssList[modRssName].advanced && Object.keys(rssList[modRssName].advanced).length > 0) {
          uniqueSettings = rssList[modRssName].advanced
        }
      }
      getFeed({ link, rssList, uniqueSettings, shardId, debugFeeds, config, feedData, scheduleName, runNum }, () => {
        if (++c === len) process.send({ status: 'batch_connected' })
      })
    }
  }).catch(err => log.general.error(`isolatedMethod db connection`, err))
})

{'fraction_non_alphanumeric': 0.1373825018076645, 'fraction_numerical': 0.0014461315979754157, 'mean_word_length': 3.4864864864864864, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7111088', 'n_tokens_mistral': 1307, 'n_tokens_neox': 1290, 'n_words': 337}