<commit_before>const TIMESTEP = 1000;
const AWS_MAX_GET_CALLS_PER_TIMESTEP = 800;

let requestTimesInTimestep: number[] = [];

export async function rateLimit() {
  const now = Date.now();
  const index = requestTimesInTimestep.findIndex(t => t > now - TIMESTEP);
  if (index === -1) return;
  requestTimesInTimestep = requestTimesInTimestep.slice(index);
  if (requestTimesInTimestep.length > AWS_MAX_GET_CALLS_PER_TIMESTEP) {
    await new Promise(resolve =>
      setTimeout(resolve, now - requestTimesInTimestep[0])
    );
  }
  requestTimesInTimestep.push(Date.now());<commit_msg>fix rate limit times filtering<commit_after>const TIMESTEP = 1000;
const AWS_MAX_GET_CALLS_PER_TIMESTEP = 800;

let requestTimesInTimestep: number[] = [];

export async function rateLimit() {
  const now = Date.now();
  const index = requestTimesInTimestep
    .slice()
    .reverse()
    .findIndex(t => t > now - TIMESTEP);
  if (index === -1) return;
  requestTimesInTimestep = requestTimesInTimestep.slice(
    requestTimesInTimestep.length - index - 1
  );
  if (requestTimesInTimestep.length > AWS_MAX_GET_CALLS_PER_TIMESTEP) {
    await new Promise(resolve =>
      setTimeout(resolve, now - requestTimesInTimestep[0])
    );
  }
  requestTimesInTimestep.push(Date.now());
{'fraction_non_alphanumeric': 0.11313291139240507, 'fraction_numerical': 0.015031645569620253, 'mean_word_length': 4.622222222222222, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5008098', 'n_tokens_mistral': 442, 'n_tokens_neox': 429, 'n_words': 85}