<commit_before>const { getAllPages, get } = require('../tools.js');
const { switchMap, bufferCount, map, tap, mergeMap, takeUntil, first } = require('rxjs/operators');
const { combineLatest, concat, BehaviorSubject, Subject } = require('rxjs');
const request = require('request');
const _cliProgress = require('cli-progress');

const langs = ['en', 'de', 'ja', 'fr'];

function prepareCache(sheet, pageName) {
  return get(`https://xivapi.com/${sheet}`).pipe(
    switchMap(firstPage => {
      return getAllPages(`https://xivapi.com/${sheet}?columns=ID,Name_*`).pipe(
        map(page => {
          return [].concat.apply([],
            page.Results.map(item => {
              return langs.map(lang => {
                return `https://beta.ffxivteamcraft.com/db/${lang}/${pageName}/${item.ID}/${item[`Name_${lang}`].split(' ').join('-')}`;
              });
            })
          );
        }),
        bufferCount(firstPage.Pagination.PageTotal),
        first(),
        map(buffered => [].concat.apply([], buffered)),
        switchMap(urls => {
          console.log(urls.length);
          // create a new progress bar instance and use shades_classic theme
          const progress = new _cliProgress.Bar({}, _cliProgress.Presets.shades_classic);
          progress.start(urls.length, 0);
          const index$ = new BehaviorSubject(0);
          const complete$ = new Subject();
          return index$.pipe(
            mergeMap(index => {
              const done$ = new Subject();
              const options = {
                url: urls[index],
                headers: {
                  'User-Agent': 'Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)'
                }
              };
              request(options, (err) => {
                if (err) {
                  console.error(err);
                }
                done$.next();<commit_msg>chore: forgot a console.log ><<commit_after>const { getAllPages, get } = require('../tools.js');
const { switchMap, bufferCount, map, tap, mergeMap, takeUntil, first } = require('rxjs/operators');
const { combineLatest, concat, BehaviorSubject, Subject } = require('rxjs');
const request = require('request');
const _cliProgress = require('cli-progress');

const langs = ['en', 'de', 'ja', 'fr'];

function prepareCache(sheet, pageName) {
  return get(`https://xivapi.com/${sheet}`).pipe(
    switchMap(firstPage => {
      return getAllPages(`https://xivapi.com/${sheet}?columns=ID,Name_*`).pipe(
        map(page => {
          return [].concat.apply([],
            page.Results.map(item => {
              return langs.map(lang => {
                return `https://beta.ffxivteamcraft.com/db/${lang}/${pageName}/${item.ID}/${item[`Name_${lang}`].split(' ').join('-')}`;
              });
            })
          );
        }),
        bufferCount(firstPage.Pagination.PageTotal),
        first(),
        map(buffered => [].concat.apply([], buffered)),
        switchMap(urls => {
          // create a new progress bar instance and use shades_classic theme
          const progress = new _cliProgress.Bar({}, _cliProgress.Presets.shades_classic);
          progress.start(urls.length, 0);
          const index$ = new BehaviorSubject(0);
          const complete$ = new Subject();
          return index$.pipe(
            mergeMap(index => {
              const done$ = new Subject();
              const options = {
                url: urls[index],
                headers: {
                  'User-Agent': 'Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)'
                }
              };
              request(options, (err) => {
                if (err) {
                  console.error(err);
                }
                done$.next();
{'fraction_non_alphanumeric': 0.16325988850544199, 'fraction_numerical': 0.003185558800106185, 'mean_word_length': 2.0362610797743756, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 20, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6636702', 'n_tokens_mistral': 1081, 'n_tokens_neox': 1059, 'n_words': 208}