<commit_before>import * as request from 'supertest'
import { Job, JobState } from '../../models'
import { wait } from '../miscs/miscs'
import { ServerInfo } from './servers'

function getJobsList (url: string, accessToken: string, state: JobState) {
  const path = '/api/v1/jobs/' + state

  return request(url)
          .get(path)
          .set('Accept', 'application/json')
          .set('Authorization', 'Bearer ' + accessToken)
          .expect(200)
          .expect('Content-Type', /json/)
}

function getJobsListPaginationAndSort (url: string, accessToken: string, state: JobState, start: number, count: number, sort: string) {
  const path = '/api/v1/jobs/' + state

  return request(url)
          .get(path)
          .query({ start })
          .query({ count })
          .query({ sort })
          .set('Accept', 'application/json')
          .set('Authorization', 'Bearer ' + accessToken)
          .expect(200)
          .expect('Content-Type', /json/)
}

async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
  let servers: ServerInfo[]

  if (Array.isArray(serversArg) === false) servers = [ serversArg as ServerInfo ]
  else servers = serversArg as ServerInfo[]

  const states: JobState[] = [ 'waiting', 'active', 'delayed' ]
  const tasks: Promise<any>[] = []
  let pendingRequests: boolean

  do {
    pendingRequests = false

    // Check if each server has pending request
    for (const server of servers) {
      for (const state of states) {
        const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt')
          .then(res => res.body.data)
          .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views'))
          .then(jobs => {
            if (jobs.length !== 0) pendingRequests = true
          })
        tasks.push(p)
      }
    }

    await Promise.all(tasks)

    // Retry, in case of new jobs were created
    if (pendingRequests === false) {
      await wait(1000)

      await Promise.all(tasks)
    }

    if (pendingRequests) {
      await wait(1000)
    }
  } while (pendingRequests)
}

// ---------------------------------------------------------------------------

export {<commit_msg>increase waitJobs pendingJobs timeout to 2000<commit_after>import * as request from 'supertest'
import { Job, JobState } from '../../models'
import { wait } from '../miscs/miscs'
import { ServerInfo } from './servers'

function getJobsList (url: string, accessToken: string, state: JobState) {
  const path = '/api/v1/jobs/' + state

  return request(url)
          .get(path)
          .set('Accept', 'application/json')
          .set('Authorization', 'Bearer ' + accessToken)
          .expect(200)
          .expect('Content-Type', /json/)
}

function getJobsListPaginationAndSort (url: string, accessToken: string, state: JobState, start: number, count: number, sort: string) {
  const path = '/api/v1/jobs/' + state

  return request(url)
          .get(path)
          .query({ start })
          .query({ count })
          .query({ sort })
          .set('Accept', 'application/json')
          .set('Authorization', 'Bearer ' + accessToken)
          .expect(200)
          .expect('Content-Type', /json/)
}

async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
  let servers: ServerInfo[]

  if (Array.isArray(serversArg) === false) servers = [ serversArg as ServerInfo ]
  else servers = serversArg as ServerInfo[]

  const states: JobState[] = [ 'waiting', 'active', 'delayed' ]
  const tasks: Promise<any>[] = []
  let pendingRequests: boolean

  do {
    pendingRequests = false

    // Check if each server has pending request
    for (const server of servers) {
      for (const state of states) {
        const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt')
          .then(res => res.body.data)
          .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views'))
          .then(jobs => {
            if (jobs.length !== 0) pendingRequests = true
          })
        tasks.push(p)
      }
    }

    await Promise.all(tasks)

    // Retry, in case of new jobs were created
    if (pendingRequests === false) {
      await wait(2000)

      await Promise.all(tasks)
    }

    if (pendingRequests) {
      await wait(1000)
    }
  } while (pendingRequests)
}

// ---------------------------------------------------------------------------

export {
{'fraction_non_alphanumeric': 0.17117117117117117, 'fraction_numerical': 0.00990990990990991, 'mean_word_length': 2.246345029239766, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7192065', 'n_tokens_mistral': 1342, 'n_tokens_neox': 1306, 'n_words': 355}