<commit_before>import _ from 'lodash';
import { LockableBuckets, StatTypes, MinMaxIgnored, MinMax } from '../types';
import { statTier } from '../utils';
import { statHashes } from '../types';
import {
  ProcessItemsByBucket,
  ProcessItem,
  ProcessArmorSet,
  IntermediateProcessArmorSet,
  LockedArmor2ProcessMods,
  ProcessMod,
} from './types';
import { DestinySocketCategoryStyle } from 'bungie-api-ts/destiny2';
import {
  canTakeAllSeasonalMods,
  sortProcessModsOrProcessItems,
  canTakeAllGeneralMods,
} from './processUtils';
import { armor2PlugCategoryHashesByName } from '../../search/d2-known-values';

const RETURNED_ARMOR_SETS = 200;

type SetTracker = {
  tier: number;
  statMixes: { statMix: string; armorSet: IntermediateProcessArmorSet }[];
}[];

function insertIntoSetTracker(
  tier: number,
  statMix: string,
  armorSet: IntermediateProcessArmorSet,
  setTracker: SetTracker
): void {
  if (setTracker.length === 0) {
    setTracker.push({ tier, statMixes: [{ statMix, armorSet }] });
    return;
  }

  for (let tierIndex = 0; tierIndex < setTracker.length; tierIndex++) {
    const currentTier = setTracker[tierIndex];

    if (tier > currentTier.tier) {
      setTracker.splice(tierIndex, 0, { tier, statMixes: [{ statMix, armorSet }] });
      return;
    }

    if (tier === currentTier.tier) {
      const currentStatMixes = currentTier.statMixes;

      for (let statMixIndex = 0; statMixIndex < currentStatMixes.length; statMixIndex++) {
        const currentStatMix = currentStatMixes[statMixIndex];

        if (statMix > currentStatMix.statMix) {
          currentStatMixes.splice(statMixIndex, 0, { statMix, armorSet });
          return;
        }

        if (currentStatMix.statMix === statMix) {
          if (armorSet.maxPower > currentStatMix.armorSet.maxPower) {
            currentStatMix.armorSet.sets = armorSet.sets.concat(currentStatMix.armorSet.sets);
            currentStatMix.armorSet.firstValidSet = armorSet.firstValidSet;
            currentStatMix.armorSet.maxPower = armorSet.maxPower;
            currentStatMix.armorSet.firstValidSetStatChoices = armorSet.firstValidSetStatChoices;
          } else {
            currentStatMix.armorSet.sets = currentStatMix.armorSet.sets.concat(armorSet.sets);
          }
          return;
        }

        if (statMixIndex === currentStatMixes.length - 1) {
          currentStatMixes.push({ statMix, armorSet });
          return;
        }
      }
    }

    if (tierIndex === setTracker.length - 1) {
      setTracker.push({ tier, statMixes: [{ statMix, armorSet }] });
      return;
    }
  }
}

/**
 * This processes all permutations of armor to build sets
 * @param filteredItems pared down list of items to process sets from
 * @param modStatTotals Stats that are applied to final stat totals, think general and seasonal mod stats
 */
export function process(
  filteredItems: ProcessItemsByBucket,
  processedSeasonalMods: ProcessMod[],
  modStatTotals: { [stat in StatTypes]: number },
  lockedArmor2ModMap: LockedArmor2ProcessMods,
  assumeMasterwork: boolean,
  statOrder: StatTypes[],
  statFilters: { [stat in StatTypes]: MinMaxIgnored },
  minimumPower: number
): {
  sets: ProcessArmorSet[];
  combos: number;
  combosWithoutCaps: number;
  statRanges?: { [stat in StatTypes]: MinMax };
} {
  const pstart = performance.now();

  processedSeasonalMods.sort(sortProcessModsOrProcessItems);

  const orderedStatValues = statOrder.map((statType) => statHashes[statType]);
  const orderedConsideredStats = statOrder.filter((statType) => !statFilters[statType].ignored);

  const statRanges: { [stat in StatTypes]: MinMax } = {
    Mobility: statFilters.Mobility.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Resilience: statFilters.Resilience.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Recovery: statFilters.Recovery.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Discipline: statFilters.Discipline.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Intellect: statFilters.Intellect.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Strength: statFilters.Strength.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
  };

  const helms = _.sortBy(filteredItems[LockableBuckets.helmet] || [], (i) => -i.basePower);
  const gaunts = _.sortBy(filteredItems[LockableBuckets.gauntlets] || [], (i) => -i.basePower);
  const chests = _.sortBy(filteredItems[LockableBuckets.chest] || [], (i) => -i.basePower);
  const legs = _.sortBy(filteredItems[LockableBuckets.leg] || [], (i) => -i.basePower);
  const classItems = _.sortBy(filteredItems[LockableBuckets.classitem] || [], (i) => -i.basePower);

  // We won't search through more than this number of stat combos - it can cause us to run out of memory.
  const combosLimit = 1_000_000;

  const combosWithoutCaps =
    helms.length * gaunts.length * chests.length * legs.length * classItems.length;

  let combos = combosWithoutCaps;

  // If we're over the limit, start trimming down the armor lists starting with the longest.
  // Since we're already sorted by total stats descending this should toss the worst items.
  while (combos > combosLimit) {
    const longestList = _.maxBy([helms, gaunts, chests, legs, classItems], (l) => l.length);
    longestList!.pop();
    combos = helms.length * gaunts.length * chests.length * legs.length * classItems.length;
  }

  if (combos < combosWithoutCaps) {
    console.log('Reduced armor combinations from', combosWithoutCaps, 'to', combos);
  }

  if (combos === 0) {
    return { sets: [], combos: 0, combosWithoutCaps: 0 };
  }

  const setTracker: SetTracker = [];

  const statsCache: Record<string, number[]> = {};

  let lowestTier = 100;
  let setCount = 0;

  for (const helm of helms) {
    const helmStats = getStatMix(helm, assumeMasterwork, orderedStatValues);
    for (const gaunt of gaunts) {
      const gauntStats = getStatMix(gaunt, assumeMasterwork, orderedStatValues);
      for (const chest of chests) {
        const chestStats = getStatMix(chest, assumeMasterwork, orderedStatValues);
        for (const leg of legs) {
          const legStats = getStatMix(leg, assumeMasterwork, orderedStatValues);
          for (const classItem of classItems) {
            const classItemStats = getStatMix(classItem, assumeMasterwork, orderedStatValues);
            const armor = [helm, gaunt, chest, leg, classItem];

            // Make sure there is at most one exotic
            if (_.sumBy(armor, (item) => (item.equippingLabel ? 1 : 0)) <= 1) {
              const statChoices = [helmStats, gauntStats, chestStats, legStats, classItemStats];

              const maxPower = getPower(armor);

              if (maxPower < minimumPower) {
                continue;
              }

              const stats: any = {};
              for (const stat of statChoices) {
                let index = 0;
                for (const key of statOrder) {
                  stats[key] = Math.min((stats[key] || 0) + stat[index], 100);
                  index++;
                }
              }

              // A string version of the tier-level of each stat, separated by commas<commit_msg>Cached the stat arrays to move them out of the hot loop.<commit_after>import _ from 'lodash';
import { LockableBuckets, StatTypes, MinMaxIgnored, MinMax } from '../types';
import { statTier } from '../utils';
import { statHashes } from '../types';
import {
  ProcessItemsByBucket,
  ProcessItem,
  ProcessArmorSet,
  IntermediateProcessArmorSet,
  LockedArmor2ProcessMods,
  ProcessMod,
} from './types';
import { DestinySocketCategoryStyle } from 'bungie-api-ts/destiny2';
import {
  canTakeAllSeasonalMods,
  sortProcessModsOrProcessItems,
  canTakeAllGeneralMods,
} from './processUtils';
import { armor2PlugCategoryHashesByName, TOTAL_STAT_HASH } from '../../search/d2-known-values';

const RETURNED_ARMOR_SETS = 200;

type SetTracker = {
  tier: number;
  statMixes: { statMix: string; armorSet: IntermediateProcessArmorSet }[];
}[];

function insertIntoSetTracker(
  tier: number,
  statMix: string,
  armorSet: IntermediateProcessArmorSet,
  setTracker: SetTracker
): void {
  if (setTracker.length === 0) {
    setTracker.push({ tier, statMixes: [{ statMix, armorSet }] });
    return;
  }

  for (let tierIndex = 0; tierIndex < setTracker.length; tierIndex++) {
    const currentTier = setTracker[tierIndex];

    if (tier > currentTier.tier) {
      setTracker.splice(tierIndex, 0, { tier, statMixes: [{ statMix, armorSet }] });
      return;
    }

    if (tier === currentTier.tier) {
      const currentStatMixes = currentTier.statMixes;

      for (let statMixIndex = 0; statMixIndex < currentStatMixes.length; statMixIndex++) {
        const currentStatMix = currentStatMixes[statMixIndex];

        if (statMix > currentStatMix.statMix) {
          currentStatMixes.splice(statMixIndex, 0, { statMix, armorSet });
          return;
        }

        if (currentStatMix.statMix === statMix) {
          if (armorSet.maxPower > currentStatMix.armorSet.maxPower) {
            currentStatMix.armorSet.sets = armorSet.sets.concat(currentStatMix.armorSet.sets);
            currentStatMix.armorSet.firstValidSet = armorSet.firstValidSet;
            currentStatMix.armorSet.maxPower = armorSet.maxPower;
            currentStatMix.armorSet.firstValidSetStatChoices = armorSet.firstValidSetStatChoices;
          } else {
            currentStatMix.armorSet.sets = currentStatMix.armorSet.sets.concat(armorSet.sets);
          }
          return;
        }

        if (statMixIndex === currentStatMixes.length - 1) {
          currentStatMixes.push({ statMix, armorSet });
          return;
        }
      }
    }

    if (tierIndex === setTracker.length - 1) {
      setTracker.push({ tier, statMixes: [{ statMix, armorSet }] });
      return;
    }
  }
}

/**
 * This processes all permutations of armor to build sets
 * @param filteredItems pared down list of items to process sets from
 * @param modStatTotals Stats that are applied to final stat totals, think general and seasonal mod stats
 */
export function process(
  filteredItems: ProcessItemsByBucket,
  processedSeasonalMods: ProcessMod[],
  modStatTotals: { [stat in StatTypes]: number },
  lockedArmor2ModMap: LockedArmor2ProcessMods,
  assumeMasterwork: boolean,
  statOrder: StatTypes[],
  statFilters: { [stat in StatTypes]: MinMaxIgnored },
  minimumPower: number
): {
  sets: ProcessArmorSet[];
  combos: number;
  combosWithoutCaps: number;
  statRanges?: { [stat in StatTypes]: MinMax };
} {
  const pstart = performance.now();

  processedSeasonalMods.sort(sortProcessModsOrProcessItems);

  const orderedStatValues = statOrder.map((statType) => statHashes[statType]);
  const orderedConsideredStats = statOrder.filter((statType) => !statFilters[statType].ignored);

  const statRanges: { [stat in StatTypes]: MinMax } = {
    Mobility: statFilters.Mobility.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Resilience: statFilters.Resilience.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Recovery: statFilters.Recovery.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Discipline: statFilters.Discipline.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Intellect: statFilters.Intellect.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
    Strength: statFilters.Strength.ignored ? { min: 0, max: 10 } : { min: 10, max: 0 },
  };

  const helms = _.sortBy(
    filteredItems[LockableBuckets.helmet] || [],
    (i) => -i.baseStats[TOTAL_STAT_HASH]
  );
  const gaunts = _.sortBy(
    filteredItems[LockableBuckets.gauntlets] || [],
    (i) => -i.baseStats[TOTAL_STAT_HASH]
  );
  const chests = _.sortBy(
    filteredItems[LockableBuckets.chest] || [],
    (i) => -i.baseStats[TOTAL_STAT_HASH]
  );
  const legs = _.sortBy(
    filteredItems[LockableBuckets.leg] || [],
    (i) => -i.baseStats[TOTAL_STAT_HASH]
  );
  const classItems = _.sortBy(
    filteredItems[LockableBuckets.classitem] || [],
    (i) => -i.baseStats[TOTAL_STAT_HASH]
  );

  // We won't search through more than this number of stat combos - it can cause us to run out of memory.
  const combosLimit = 2_000_000;

  const combosWithoutCaps =
    helms.length * gaunts.length * chests.length * legs.length * classItems.length;

  let combos = combosWithoutCaps;

  // If we're over the limit, start trimming down the armor lists starting with the longest.
  // Since we're already sorted by total stats descending this should toss the worst items.
  while (combos > combosLimit) {
    const longestList = _.maxBy([helms, gaunts, chests, legs, classItems], (l) => l.length);
    longestList!.pop();
    combos = helms.length * gaunts.length * chests.length * legs.length * classItems.length;
  }

  if (combos < combosWithoutCaps) {
    console.log('Reduced armor combinations from', combosWithoutCaps, 'to', combos);
  }

  if (combos === 0) {
    return { sets: [], combos: 0, combosWithoutCaps: 0 };
  }

  const setTracker: SetTracker = [];

  let lowestTier = 100;
  let setCount = 0;

  const statsCache: Record<string, number[]> = {};

  for (const item of [...helms, ...gaunts, ...chests, ...legs, ...classItems]) {
    statsCache[item.id] = getStatMix(item, assumeMasterwork, orderedStatValues);
  }

  for (const helm of helms) {
    for (const gaunt of gaunts) {
      for (const chest of chests) {
        for (const leg of legs) {
          for (const classItem of classItems) {
            const armor = [helm, gaunt, chest, leg, classItem];

            // Make sure there is at most one exotic
            if (_.sumBy(armor, (item) => (item.equippingLabel ? 1 : 0)) <= 1) {
              const statChoices = [
                statsCache[helm.id],
                statsCache[gaunt.id],
                statsCache[chest.id],
                statsCache[leg.id],
                statsCache[classItem.id],
              ];

              const maxPower = getPower(armor);

              if (maxPower < minimumPower) {
                continue;
              }

              const stats: any = {};
              for (const stat of statChoices) {
                let index = 0;
                for (const key of statOrder) {
                  stats[key] = Math.min((stats[key] || 0) + stat[index], 100);
                  index++;
                }
              }

              // A string version of the tier-level of each stat, separated by commas
{'fraction_non_alphanumeric': 0.12012220524927093, 'fraction_numerical': 0.010276350506874045, 'mean_word_length': 3.0186941964285716, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5824118', 'n_tokens_mistral': 4722, 'n_tokens_neox': 4466, 'n_words': 1126}