<commit_before>import { all, call, put, take } from "redux-saga/effects";
import {
  fetchMetadataEntries,
  saveMetadataEntryField,
} from "../../../../apis/metadata/entry";
import { actions, types } from "../reducers/entries";
import { FIELDS } from "../constants";

/**
 * Fetch all the metadata entries required to initialize the table.
 * @returns {IterableIterator<*>}
 */
export function* entriesLoadingSaga() {
  let pageSize = 1000,
    promises = [],
    count;
  try {
    const { payload } = yield take('METADATA_TEMPLATES_LOADED');
    yield put(actions.load());
    const pages = Math.ceil(window.PAGE.totalSamples / pageSize);

    for (let i = 0; i < pages; i++) {
      promises.push(call(fetchEntrySet, payload.id, i, pageSize));
    }
    const entries = yield all(promises);

    yield put(actions.success({ entries: entries.flat() }));
  } catch (error) {<commit_msg>Calculate the best page size for requesting entries<commit_after>import { all, call, put, take } from "redux-saga/effects";
import {
  fetchMetadataEntries,
  saveMetadataEntryField,
} from "../../../../apis/metadata/entry";
import { actions, types } from "../reducers/entries";
import { FIELDS } from "../constants";

/**
 * Calculate the best page size for fetching entries
 * @returns size of page of entries to request
 */
function calculatePageSize() {
  const MIN = 1000;
  const MAX = 5000;
  const estimated = window.PAGE.totalSamples / 100;
  return estimated < MIN ? MIN : estimated > MAX ? MAX : estimated;
}

/**
 * Fetch all the metadata entries required to initialize the table.
 * @returns {IterableIterator<*>}
 */
export function* entriesLoadingSaga() {
  let pageSize = calculatePageSize(),
    promises = [],
    count;
  try {
    const { payload } = yield take("METADATA_TEMPLATES_LOADED");
    yield put(actions.load());
    const pages = Math.ceil(window.PAGE.totalSamples / pageSize);

    for (let i = 0; i < pages; i++) {
      promises.push(call(fetchEntrySet, payload.id, i, pageSize));
    }
    const entries = yield all(promises);

    yield put(actions.success({ entries: entries.flat() }));
  } catch (error) {
{'fraction_non_alphanumeric': 0.14547160957297042, 'fraction_numerical': 0.007977475363679024, 'mean_word_length': 3.565310492505353, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4265161', 'n_tokens_mistral': 658, 'n_tokens_neox': 630, 'n_words': 189}