<commit_before>/**

 In this example of ~1000ms data loading tasks, using data-processor can
 speed up 31555 times compare to one-by-one sync version
 archived ~25% of 'pure processing' performance

 |===
 | mode | average TPS* |

 | sync version with delay
 |      1.77

 | data-processor pipeline
 |  55872.16

 | sync version without delay
 | 222816.40

 |===
 *: rounded to 2 decimal places

 * */
import { later } from '../src/async/wait';
import { format_time_duration } from '../src/format';
import { catchMain } from '../src/node';
import { batchProcess } from '../src/task/data-processor';

let clearLine = () => process.stdout.write('\r' + ' '.repeat(32) + '\r');

let n = 1000 * 1000;
let delay = 1000;
let keys = new Array(n).fill(0).map((x, i) => i);
let startTime = Date.now();
let last = -1;
let delays = keys.map(() => delay * Math.random());

console.log('benchmarking data-processor pipeline');
catchMain(
  batchProcess({
    maxConcurrent: 1000 * 100,
    keys,
    loader: key => later(delays[key]).then(() => {
      return ({
        key,
        value: key.toString().repeat(10000),
      });
    }),
    processor: ({ key, value }) => {
      if (key < last) {
        clearLine();
        console.error({ last, key });
        throw new Error('not processing in order');
      }
      last = key;
      process.stdout.write('\rprocessing: ' + key);
    },
  }).then(() => {
    clearLine();
    let endTime = Date.now();
    let usedTime = endTime - startTime;
    console.log('used: ', format_time_duration(usedTime));
    console.log('average TPS: ' + (n / (usedTime / 1000)));
    return rawSyncTest(usedTime);
  }));

let rawSyncTest = (async (duration: number) => {
  console.log();
  console.log('benchmarking sync version with delay');<commit_msg>fixed tsc error in data-processor-test.ts<commit_after>/**

 In this example of ~1000ms data loading tasks, using data-processor can
 speed up 31555 times compare to one-by-one sync version
 archived ~25% of 'pure processing' performance

 |===
 | mode | average TPS* |

 | sync version with delay
 |      1.77

 | data-processor pipeline
 |  55872.16

 | sync version without delay
 | 222816.40

 |===
 *: rounded to 2 decimal places

 * */
import { later } from '../src/async/wait';
import { format_time_duration } from '../src/format';
import { catchMain } from '../src/node';
import { batchProcess } from '../src/task/data-processor';

let clearLine = () => process.stdout.write('\r' + ' '.repeat(32) + '\r');

let n = 1000 * 1000;
let delay = 1000;
let keys = new Array(n).fill(0).map((x, i) => i);
let startTime = Date.now();
let last = -1;
let delays = keys.map(() => delay * Math.random());

console.log('benchmarking data-processor pipeline');
catchMain(
  batchProcess<number, { key: number, value: string }>({
    maxConcurrent: 1000 * 100,
    keys,
    loader: (key: number) => later(delays[key]).then(() => {
      return ({
        key,
        value: key.toString().repeat(10000),
      });
    }),
    processor: ({ key, value }) => {
      if (key < last) {
        clearLine();
        console.error({ last, key });
        throw new Error('not processing in order');
      }
      last = key;
      process.stdout.write('\rprocessing: ' + key);
    },
  }).then(() => {
    clearLine();
    let endTime = Date.now();
    let usedTime = endTime - startTime;
    console.log('used: ', format_time_duration(usedTime));
    console.log('average TPS: ' + (n / (usedTime / 1000)));
    return rawSyncTest(usedTime);
  }));

let rawSyncTest = (async (duration: number) => {
  console.log();
  console.log('benchmarking sync version with delay');
{'fraction_non_alphanumeric': 0.16987355689939526, 'fraction_numerical': 0.034084661902144035, 'mean_word_length': 2.9770491803278687, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4411786', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1120, 'n_words': 324}