<commit_before>import sizeof from 'object-sizeof';

let options = {};
let cache = {};
let ttlQueue = [];
let ttlExtend = new Set();

const genExpire = seconds => {
  const t = new Date();
  t.setSeconds(t.getSeconds() + seconds);
  return t;
};

const addToTTLQueue = ttl => {
  ttlQueue = ttlQueue.filter(e => e.id !== ttl.id);
  for (let i = 0; i < ttlQueue.length; i++) {
    if (ttl.expires.getTime() < ttlQueue[i].expires.getTime()) {
      ttlQueue.splice(i, 0, ttl);
      return;
    }
  }
  ttlQueue.push(ttl);
};

const cleanExpired = () => {
  if (ttlQueue.length === 0) return;
  const now = new Date().getTime();
  if (ttlQueue[0].expires.getTime() > now) return;
  for (let i = 0; i < ttlQueue.length; i++) {
    if (now < ttlQueue[i].expires.getTime()) {
      ttlQueue.splice(0, i);
      return;
    }
    delete cache[ttlQueue[i].id];
  }
  ttlQueue = [];
};
<commit_msg>Improve ttlQueue insertion<commit_after>import sizeof from 'object-sizeof';

let options = {};
let cache = {};
let ttlQueue = [];
let ttlExtend = new Set();

const genExpire = seconds => {
  const t = new Date();
  t.setSeconds(t.getSeconds() + seconds);
  return t;
};

const sortedIndex = (array, value) => {
  let low = 0;
  let high = array.length;

  while (low < high) {
    const mid = low + high >>> 1; // eslint-disable-line no-bitwise
    if (array[mid].expires.getTime() < value) low = mid + 1;
    else high = mid;
  }
  return low;
};

const addToTTLQueue = ttl => {
  ttlQueue = ttlQueue.filter(e => e.id !== ttl.id);
  ttlQueue.splice(sortedIndex(ttlQueue, ttl.expires.getTime()), 0, ttl);
};

const cleanExpired = () => {
  if (ttlQueue.length === 0) return;
  const now = new Date().getTime();
  if (ttlQueue[0].expires.getTime() > now) return;
  for (let i = 0; i < ttlQueue.length; i++) {
    if (now < ttlQueue[i].expires.getTime()) {
      ttlQueue.splice(0, i);
      return;
    }
    delete cache[ttlQueue[i].id];
  }
  ttlQueue = [];
};

{'fraction_non_alphanumeric': 0.18195797027165556, 'fraction_numerical': 0.007175807278318811, 'mean_word_length': 3.1443736730360934, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '50923', 'n_tokens_mistral': 760, 'n_tokens_neox': 746, 'n_words': 176}