<commit_before>import {chunkify} from './chunkify';
import {parseOptions} from './options';
import {expect} from 'chai';
import {spy} from 'sinon';


describe('chunkify', () => {

  // FIXME (?): these tests are non-deterministic
  const DELAY = 10;
  const TOLERANCE = 5;

  it('should throw when not given a number "start" index', () => {
    for (let start of [undefined, null, 'string', {}, []]) {
      expect(() => { chunkify(start); })
        .throws(/start index "start" of generator range must be a number/);
    }
  });

  it('should throw when not given a number "final" index', () => {
    for (let final of [undefined, null, 'string', {}, []]) {
      expect(() => { chunkify(0, final); })
        .throws(/final index "final" of generator range must be a number/);
    }
  });

  it('should yield Promise<void> after "chunk" iterations resolving in "delay" milliseconds', done => {
    let iter = chunkify(0, 3, {chunk: 2, delay: DELAY});

    expect(iter.next()).to.deep.equal({done: false, value: 0});
    expect(iter.next()).to.deep.equal({done: false, value: 1});
    
    let now = (): number => {
      return new Number(new Date).valueOf();
    }

    let timeout: number | Promise<void> = iter.next().value;
    let started: number = now();
    let elapsed: number;

    expect(timeout instanceof Promise).to.be.ok;

    timeout
      .then(() => {
        elapsed = now() - started;
        expect(elapsed >= DELAY).to.be.ok;
        expect(elapsed <= DELAY + TOLERANCE).to.be.ok;
        expect(iter.next()).to.deep.equal({done: false, value: 2});
      })
      .then(done);
  });

  // test('should throw an error if advanced while delay is pending', () => {
  //   let it = chunkify(0, 3, {chunk: 2, delay: 100});

  //   it.next();
  //   it.next();
  //   it.next();  // enters paused state

  //   t.throws(() => {
  //     it.next();
  //   }, /paused at index 2; wait 100 milliseconds/);
  //   t.end()
  // });
<commit_msg>catch async errors<commit_after>import {chunkify} from './chunkify';
import {parseOptions} from './options';
import {expect} from 'chai';
import {spy} from 'sinon';


describe('chunkify', () => {

  // FIXME (?): these tests are non-deterministic
  const DELAY = 10;
  const TOLERANCE = 5;

  it('should throw when not given a number "start" index', () => {
    for (let start of [undefined, null, 'string', {}, []]) {
      expect(() => { chunkify(start); })
        .throws(/start index "start" of generator range must be a number/);
    }
  });

  it('should throw when not given a number "final" index', () => {
    for (let final of [undefined, null, 'string', {}, []]) {
      expect(() => { chunkify(0, final); })
        .throws(/final index "final" of generator range must be a number/);
    }
  });

  it('should yield Promise<void> after "chunk" iterations resolving in "delay" milliseconds', done => {
    let iter = chunkify(0, 3, {chunk: 2, delay: DELAY});

    expect(iter.next()).to.deep.equal({done: false, value: 0});
    expect(iter.next()).to.deep.equal({done: false, value: 1});
    
    let now = (): number => {
      return new Number(new Date).valueOf();
    }

    let timeout: number | Promise<void> = iter.next().value;
    let started: number = now();
    let elapsed: number;

    expect(timeout instanceof Promise).to.be.ok;

    timeout
      .then(() => {
        elapsed = now() - started;
        expect(elapsed >= DELAY).to.be.ok;
        expect(elapsed <= DELAY + TOLERANCE).to.be.ok;
        expect(iter.next()).to.deep.equal({done: false, value: 2});
      })
      .then(done)
      .catch(done);
  });

  // test('should throw an error if advanced while delay is pending', () => {
  //   let it = chunkify(0, 3, {chunk: 2, delay: 100});

  //   it.next();
  //   it.next();
  //   it.next();  // enters paused state

  //   t.throws(() => {
  //     it.next();
  //   }, /paused at index 2; wait 100 milliseconds/);
  //   t.end()
  // });

{'fraction_non_alphanumeric': 0.18918232605383442, 'fraction_numerical': 0.01015744032503809, 'mean_word_length': 2.6371191135734073, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5408154', 'n_tokens_mistral': 1332, 'n_tokens_neox': 1251, 'n_words': 369}