<commit_before>import { Item } from './item';
import { Settings } from '../classes/settings';

export class ItemCache {
  $index: number;
  nodeId: string;
  data: any; // todo: cache data only if it is permitted by settings
  size: number;
  position: number;

  constructor(item: Item) {
    this.$index = item.$index;
    this.nodeId = item.nodeId;
    this.data = item.data;
    this.size = item.size;
  }
}

export class Cache {
  items: Array<ItemCache>;
  averageSize: number;
  minIndex: number;
  maxIndex: number;

  constructor(settings: Settings) {
    this.items = [];
    this.averageSize = settings.itemSize;
    this.resetIndexes();
  }

  resetIndexes() {
    this.minIndex = +Infinity;
    this.maxIndex = -Infinity;
  }

  updateAverageSize(item: ItemCache) {
    this.averageSize = ((this.items.length - 1) * this.averageSize + item.size) / this.items.length;
  }

  add(item: Item): ItemCache {
    // const isForward = direction !== Direction.backward;
    let itemCache = this.items.find(i => i.$index === item.$index);
    if (itemCache) {
      itemCache.data = item.data;
      if (itemCache.size !== item.size) {
        itemCache.size = item.size;
        this.updateAverageSize(itemCache);
        // todo: update positions ?
      }
    } else {
      // todo: do we need the list to be sorted? maybe an object?
      itemCache = new ItemCache(item);
      this.items.push(itemCache);
      if (this.averageSize !== itemCache.size) {
        this.updateAverageSize(itemCache);
      }
    }
    if (item.$index < this.minIndex) {
      this.minIndex = item.$index;
    }
    if (item.$index > this.maxIndex) {
      this.maxIndex = item.$index;
    }
    return itemCache;
  }

  get(index: number): ItemCache | undefined {
    return this.items.find((item: ItemCache) => item.$index === index);
  }
}<commit_msg>replace Cache Array with Map<commit_after>import { Item } from './item';
import { Settings } from '../classes/settings';

export class ItemCache {
  $index: number;
  nodeId: string;
  data: any; // todo: cache data only if it is permitted by settings
  size: number;
  position: number;

  constructor(item: Item) {
    this.$index = item.$index;
    this.nodeId = item.nodeId;
    this.data = item.data;
    this.size = item.size;
  }
}

export class Cache {
  averageSize: number;
  minIndex: number;
  maxIndex: number;

  private items: Map<number, ItemCache>;

  constructor(settings: Settings) {
    this.items = new Map<number, ItemCache>();
    this.averageSize = settings.itemSize;
    this.resetIndexes();
  }

  resetIndexes() {
    this.minIndex = +Infinity;
    this.maxIndex = -Infinity;
  }

  updateAverageSize(item: ItemCache) {
    this.averageSize = ((this.items.size - 1) * this.averageSize + item.size) / this.items.size;
  }

  add(item: Item): ItemCache {
    let itemCache = this.items.get(item.$index);
    if (itemCache) {
      itemCache.data = item.data;
      if (itemCache.size !== item.size) {
        itemCache.size = item.size;
        this.updateAverageSize(itemCache);
        // todo: update positions ?
      }
    } else {
      itemCache = new ItemCache(item);
      this.items.set(item.$index, itemCache);
      if (this.averageSize !== itemCache.size) {
        this.updateAverageSize(itemCache);
      }
    }
    if (item.$index < this.minIndex) {
      this.minIndex = item.$index;
    }
    if (item.$index > this.maxIndex) {
      this.maxIndex = item.$index;
    }
    return itemCache;
  }

  get(index: number): ItemCache | undefined {
    return this.items.get(index);
  }
}
{'fraction_non_alphanumeric': 0.1381726457399103, 'fraction_numerical': 0.0005605381165919282, 'mean_word_length': 2.7177083333333334, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1339143', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1136, 'n_words': 265}