<commit_before>import {
  resumeNotifications,
  loadUserOptions,
} from './options'

import {
  all,
  call,
  put,
  race,
  select,
  spawn,
  take,
  takeEvery,
  takeLatest,
  throttle,
} from 'redux-saga/effects'

import * as Actions from '_/actions'
import * as C from '_/constants'

import AppConfiguration from '_/config'
import { isNumber } from '_/utils'
import { delay } from './utils'

import {
  fetchAndPutSingleVm,
  fetchByPage,
  fetchPools,
  fetchSinglePool,
  fetchSingleVm,
  fetchVms,
} from './index'
import { getConsoleOptions } from './console'
import { fetchIsoFiles } from './storageDomains'
import { fetchUnknownIcons } from './osIcons'

const BACKGROUND_REFRESH = 'BACKGROUND_REFRESH'

//
// ** User Action Sagas **
//
/**
 * Change the background refresh type based on the page type, and force a refresh.
 *
 * This should be done at the time of navigation to the page, typically by the page router.
 */
function* changePage ({ payload: { type, id } }) {
  const targetPage = { type, id }
  yield put(Actions.setCurrentPage(targetPage))
  yield put(backgroundRefreshAction('changePage', targetPage))
}

/**
 * Refresh the current page NOW.
 */
function* refreshManually () {
  const targetPage = yield select(({ config }) => config.get('currentPage'))
  yield put(backgroundRefreshAction('manual', targetPage))
}

/**
 * Start or restart the background refresh timer, picking up the current refresh
 * interval from the store.
 */
function* restartBackgroundRefreshTimer () {
  yield put(backgroundRefreshAction('restartBackgroundRefreshTimer'))
}

/**
 * When ovirt-web-ui is installed to ovirt-engine, a logout should push the user to the
 * base ovirt welcome page.  But when running in dev mode or via container, the logout
 * page is displayed.  In that case, we want to make sure the page is set to something
 * appropriate and that background refreshing is no longer done.
 */
function* logoutAndCancelScheduler () {
  yield put(Actions.setCurrentPage({ type: C.NO_REFRESH_TYPE }))
  yield put(backgroundRefreshAction('stop'))
  yield put(Actions.cancelResumeNotificationsTimer())
}

//
// ** Refresh Sagas **
//
/**
 * Invoke the correct refresh function based on the provided page.
 */
function* refreshDataForTargetPage ({
  pageRouterRefresh = false,
  schedulerRefresh = false,
  manualRefresh = false,
  targetPage,
}) {
  const pageRefreshFunction = pagesRefreshers[targetPage.type]

  console.log('🔄 refreshDataForCurrentPage() 🡒 start, targetPage:', targetPage, 'pageRefreshFunction?', !!pageRefreshFunction)
  if (pageRefreshFunction) {
    yield call(pageRefreshFunction, {
      id: targetPage.id,
      pageRouterRefresh,
      schedulerRefresh,
      manualRefresh,
    })
  }
  yield put(Actions.updateLastRefresh())
  console.log('🔄 refreshDataForCurrentPage() 🡒 finished')
}

const pagesRefreshers = {
  [C.NO_REFRESH_TYPE]: null,
  [C.LIST_PAGE_TYPE]: refreshListPage,
  [C.DETAIL_PAGE_TYPE]: refreshDetailPage,
  [C.CREATE_PAGE_TYPE]: refreshCreatePage,
  [C.CONSOLE_PAGE_TYPE]: refreshConsolePage,
  [C.SETTINGS_PAGE_TYPE]: loadUserOptions,
}

function* refreshListPage () {
  const {
    vmsPage,
    vmsExpectMorePages,
    existingVmIds,
    poolsPage,
    poolsExpectMorePages,
    existingPoolIds,
  } = yield select(({ vms }) => ({
    vmsPage: vms.get('vmsPage'),
    vmsExpectMorePages: !!vms.get('vmsExpectMorePages'),
    existingVmIds: Array.from(vms.get('vms').keys()),
    poolsPage: vms.get('poolsPage'),
    poolsExpectMorePages: !!vms.get('poolsExpectMorePages'),
    existingPoolIds: Array.from(vms.get('pools').keys()),
  }))

  // list page initial state - fetch the first page
  if (vmsPage === 0 && vmsExpectMorePages && poolsPage === 0 && poolsExpectMorePages) {
    yield fetchByPage()
    return
  }

  const [vmsResults, poolsResults] = yield all([
    call(function* () {
      // fetch the VMs we are expecting to be in the pages we have fetched
      const { internalVms: expectedVms } = yield fetchVms(Actions.getVmsByCount({
        count: vmsPage * AppConfiguration.pageLimit,
      }))

      // if any existing VMs are not in expectedVms, fetch them individually
      const expectedVmIds = new Set(expectedVms.map(vm => vm.id))
      const unexpectedVms = yield all(
        existingVmIds
          .filter(vmId => !expectedVmIds.has(vmId))
          .map(vmId => call(fetchSingleVm, Actions.getSingleVm({ vmId, shallowFetch: true })))
      )

      return {
        refreshedVms: [
          ...expectedVms,
          ...unexpectedVms.filter(result => !result.error).map(result => result.internalVm),
        ],
        missedVmIds: unexpectedVms.filter(result => result.error).map(result => result.vmId),
      }
    }),

    call(function* () {
      // fetch the Pools we are expecting to be in the pages we have fetched
      const { internalPools: expectedPools } = yield fetchPools(Actions.getPoolsByCount({
        count: poolsPage * AppConfiguration.pageLimit,
      }))

      // if any existing VMs are not in expectedVms, fetch them individually
      const expectedPoolIds = new Set(expectedPools.map(pool => pool.id))
      const unexpectedPools = yield all(
        existingPoolIds
          .filter(poolId => !expectedPoolIds.has(poolId))
          .map(poolId => call(fetchSinglePool, Actions.getSinglePool({ poolId })))
      )

      return {
        refreshedPools: [
          ...expectedPools,
          ...unexpectedPools.filter(result => !result.error).map(result => result.internalPool),
        ],
        missedPoolIds: unexpectedPools.filter(result => result.error).map(result => result.poolId),
      }
    }),
  ])

  // Put the refreshed VMs and pools to the store
  yield put(Actions.updateVms({
    keepSubResources: true,
    vms: vmsResults.refreshedVms,
    removeVmIds: vmsResults.missedVmIds,

    pools: poolsResults.refreshedPools,
    removePoolIds: poolsResults.missedPoolIds,
  }))
  yield fetchUnknownIcons({ vms: vmsResults.refreshedVms })
}
<commit_msg>Handle bad vm/pool by count fetch in `refreshListPage`<commit_after>import {
  resumeNotifications,
  loadUserOptions,
} from './options'

import {
  all,
  call,
  put,
  race,
  select,
  spawn,
  take,
  takeEvery,
  takeLatest,
  throttle,
} from 'redux-saga/effects'

import * as Actions from '_/actions'
import * as C from '_/constants'

import AppConfiguration from '_/config'
import { isNumber } from '_/utils'
import { delay } from './utils'

import {
  fetchAndPutSingleVm,
  fetchByPage,
  fetchPools,
  fetchSinglePool,
  fetchSingleVm,
  fetchVms,
} from './index'
import { getConsoleOptions } from './console'
import { fetchIsoFiles } from './storageDomains'
import { fetchUnknownIcons } from './osIcons'

const BACKGROUND_REFRESH = 'BACKGROUND_REFRESH'

//
// ** User Action Sagas **
//
/**
 * Change the background refresh type based on the page type, and force a refresh.
 *
 * This should be done at the time of navigation to the page, typically by the page router.
 */
function* changePage ({ payload: { type, id } }) {
  const targetPage = { type, id }
  yield put(Actions.setCurrentPage(targetPage))
  yield put(backgroundRefreshAction('changePage', targetPage))
}

/**
 * Refresh the current page NOW.
 */
function* refreshManually () {
  const targetPage = yield select(({ config }) => config.get('currentPage'))
  yield put(backgroundRefreshAction('manual', targetPage))
}

/**
 * Start or restart the background refresh timer, picking up the current refresh
 * interval from the store.
 */
function* restartBackgroundRefreshTimer () {
  yield put(backgroundRefreshAction('restartBackgroundRefreshTimer'))
}

/**
 * When ovirt-web-ui is installed to ovirt-engine, a logout should push the user to the
 * base ovirt welcome page.  But when running in dev mode or via container, the logout
 * page is displayed.  In that case, we want to make sure the page is set to something
 * appropriate and that background refreshing is no longer done.
 */
function* logoutAndCancelScheduler () {
  yield put(Actions.setCurrentPage({ type: C.NO_REFRESH_TYPE }))
  yield put(backgroundRefreshAction('stop'))
  yield put(Actions.cancelResumeNotificationsTimer())
}

//
// ** Refresh Sagas **
//
/**
 * Invoke the correct refresh function based on the provided page.
 */
function* refreshDataForTargetPage ({
  pageRouterRefresh = false,
  schedulerRefresh = false,
  manualRefresh = false,
  targetPage,
}) {
  const pageRefreshFunction = pagesRefreshers[targetPage.type]

  console.log('🔄 refreshDataForCurrentPage() 🡒 start, targetPage:', targetPage, 'pageRefreshFunction?', !!pageRefreshFunction)
  if (pageRefreshFunction) {
    yield call(pageRefreshFunction, {
      id: targetPage.id,
      pageRouterRefresh,
      schedulerRefresh,
      manualRefresh,
    })
  }
  yield put(Actions.updateLastRefresh())
  console.log('🔄 refreshDataForCurrentPage() 🡒 finished')
}

const pagesRefreshers = {
  [C.NO_REFRESH_TYPE]: null,
  [C.LIST_PAGE_TYPE]: refreshListPage,
  [C.DETAIL_PAGE_TYPE]: refreshDetailPage,
  [C.CREATE_PAGE_TYPE]: refreshCreatePage,
  [C.CONSOLE_PAGE_TYPE]: refreshConsolePage,
  [C.SETTINGS_PAGE_TYPE]: loadUserOptions,
}

function* refreshListPage () {
  const {
    vmsPage,
    vmsExpectMorePages,
    existingVmIds,
    poolsPage,
    poolsExpectMorePages,
    existingPoolIds,
  } = yield select(({ vms }) => ({
    vmsPage: vms.get('vmsPage'),
    vmsExpectMorePages: !!vms.get('vmsExpectMorePages'),
    existingVmIds: Array.from(vms.get('vms').keys()),
    poolsPage: vms.get('poolsPage'),
    poolsExpectMorePages: !!vms.get('poolsExpectMorePages'),
    existingPoolIds: Array.from(vms.get('pools').keys()),
  }))

  // list page initial state - fetch the first page
  if (vmsPage === 0 && vmsExpectMorePages && poolsPage === 0 && poolsExpectMorePages) {
    yield fetchByPage()
    return
  }

  const [vmsResults, poolsResults] = yield all([
    call(function* () {
      // fetch the VMs we are expecting to be in the pages we have fetched
      const { internalVms: expectedVms } = yield fetchVms(Actions.getVmsByCount({
        count: vmsPage * AppConfiguration.pageLimit,
      }))

      // if any existing VMs are not in expectedVms, fetch them individually
      const expectedVmIds = new Set(expectedVms ? expectedVms.map(vm => vm.id) : [])
      const unexpectedVms = yield all(
        existingVmIds
          .filter(vmId => !expectedVmIds.has(vmId))
          .map(vmId => call(fetchSingleVm, Actions.getSingleVm({ vmId, shallowFetch: true })))
      )

      return {
        refreshedVms: [
          ...(expectedVms || []),
          ...unexpectedVms.filter(result => !result.error).map(result => result.internalVm),
        ],
        missedVmIds: unexpectedVms.filter(result => result.error).map(result => result.vmId),
      }
    }),

    call(function* () {
      // fetch the Pools we are expecting to be in the pages we have fetched
      const { internalPools: expectedPools } = yield fetchPools(Actions.getPoolsByCount({
        count: poolsPage * AppConfiguration.pageLimit,
      }))

      // if any existing VMs are not in expectedVms, fetch them individually
      const expectedPoolIds = new Set(expectedPools ? expectedPools.map(pool => pool.id) : [])
      const unexpectedPools = yield all(
        existingPoolIds
          .filter(poolId => !expectedPoolIds.has(poolId))
          .map(poolId => call(fetchSinglePool, Actions.getSinglePool({ poolId })))
      )

      return {
        refreshedPools: [
          ...(expectedPools || []),
          ...unexpectedPools.filter(result => !result.error).map(result => result.internalPool),
        ],
        missedPoolIds: unexpectedPools.filter(result => result.error).map(result => result.poolId),
      }
    }),
  ])

  // Put the refreshed VMs and pools to the store
  yield put(Actions.updateVms({
    keepSubResources: true,
    vms: vmsResults.refreshedVms,
    removeVmIds: vmsResults.missedVmIds,

    pools: poolsResults.refreshedPools,
    removePoolIds: poolsResults.missedPoolIds,
  }))
  yield fetchUnknownIcons({ vms: vmsResults.refreshedVms })
}

{'fraction_non_alphanumeric': 0.11662941809452859, 'fraction_numerical': 0.00033109841900504924, 'mean_word_length': 3.571320469163829, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4988527', 'n_tokens_mistral': 3756, 'n_tokens_neox': 3689, 'n_words': 945}