<commit_before>import filterAsync from 'node-filter-async';
import { throwIfFalsy } from 'throw-if-arg-empty';
import * as colors from 'ansi-colors';
import log from './log';
import * as ProgressBar from 'progress';
import { performance } from 'perf_hooks';
import * as prettyMS from 'pretty-ms';

export interface DataObject {
  [key: string]: unknown;
}

export type MapFn = (obj: DataObject) => Promise<DataObject>;
export type ForEachFn = (obj: DataObject) => Promise<void>;
export type FilterFn = (obj: DataObject) => Promise<boolean>;
export type ResetFn = (objs: DataObject[]) => Promise<DataObject[]>;

export default class DataList {
  static logging = true;

  static all(values: Array<unknown>, key: string): DataList {
    if (!values) {
      return new DataList();
    }
    return new DataList(values.map(value => ({ [key]: value })));
  }

  list: DataObject[];
  // defaults to -1 (not set)
  private prevLength = -1;
  private autoLog = false;
  private startTime = 0;

  constructor(list?: DataObject[], autoLog = false) {
    this.autoLog = autoLog;
    this.list = list || [];
    const name = 'Job started';
    this.onActionStarted(name);
    this.onActionEnded(name);
  }

  get count(): number {
    return this.list.length;
  }

  values(key: string): unknown {
    throwIfFalsy(key, 'key');
    return this.list.map(d => d[key]);
  }

  async map(name: string, fn: MapFn): Promise<void> {
    throwIfFalsy(fn, 'fn');
    this.onActionStarted(name);

    const promises = this.progressive(this.list.map(fn));
    this.list = await Promise.all(promises);

    this.onActionEnded(name);
  }

  async reset(name: string, fn: ResetFn): Promise<void> {
    throwIfFalsy(fn, 'fn');
    this.onActionStarted(name);

    this.list = await fn(this.list);

    this.onActionEnded(name);
  }

  async filter(name: string, fn: FilterFn): Promise<void> {
    throwIfFalsy(fn, 'fn');
    this.onActionStarted(name);

    const progBar = this.progressBar();
    this.list = await filterAsync(this.list, fn, () => progBar.tick());

    this.onActionEnded(name);
  }

  async forEach(name: string, fn: ForEachFn): Promise<void> {
    throwIfFalsy(fn, 'fn');
    this.onActionStarted(name);

    const promises = this.progressive(this.list.map(fn));
    await Promise.all(promises);

    this.onActionEnded(name);
  }

  logList() {
    // tslint:disable-next-line no-console
    console.log(this.list);
  }

  startLogging() {
    this.autoLog = true;
  }

  stopLogging() {
    this.autoLog = false;
  }

  private onActionStarted(name: string) {
    this.logName(name);
    this.startTime = performance.now();
  }

  private onActionEnded(_: string) {
    this.logLengthIfNeeded();
    this.logIfListNeeded();
    const duration = performance.now() - this.startTime;
    this.logTime(prettyMS(duration));
  }

  private logIfListNeeded() {
    if (this.autoLog) {
      this.logList();
    }
  }

  private logName(name: string) {
    throwIfFalsy(name, 'name');
    log(colors.cyan(`🦁 ${name}`));
  }

  private logTime(s: string) {
    log(colors.gray(`> Done in ${s}`));
  }

  private logLengthIfNeeded() {
    if (this.prevLength !== this.list.length) {
      const msg =
        this.prevLength >= 0
          ? `${this.prevLength} -> ${this.list.length}`
          : `${this.list.length}`;
      log(colors.yellow(`  >> ${msg}`));
      this.prevLength = this.list.length;
    }
  }

  private progressBar(): ProgressBar {<commit_msg>Tweak log length content<commit_after>import filterAsync from 'node-filter-async';
import { throwIfFalsy } from 'throw-if-arg-empty';
import * as colors from 'ansi-colors';
import log from './log';
import * as ProgressBar from 'progress';
import { performance } from 'perf_hooks';
import * as prettyMS from 'pretty-ms';

export interface DataObject {
  [key: string]: unknown;
}

export type MapFn = (obj: DataObject) => Promise<DataObject>;
export type ForEachFn = (obj: DataObject) => Promise<void>;
export type FilterFn = (obj: DataObject) => Promise<boolean>;
export type ResetFn = (objs: DataObject[]) => Promise<DataObject[]>;

export default class DataList {
  static logging = true;

  static all(values: Array<unknown>, key: string): DataList {
    if (!values) {
      return new DataList();
    }
    return new DataList(values.map(value => ({ [key]: value })));
  }

  list: DataObject[];
  // defaults to -1 (not set)
  private prevLength = -1;
  private autoLog = false;
  private startTime = 0;

  constructor(list?: DataObject[], autoLog = false) {
    this.autoLog = autoLog;
    this.list = list || [];
    const name = 'Job started';
    this.onActionStarted(name);
    this.onActionEnded(name);
  }

  get count(): number {
    return this.list.length;
  }

  values(key: string): unknown {
    throwIfFalsy(key, 'key');
    return this.list.map(d => d[key]);
  }

  async map(name: string, fn: MapFn): Promise<void> {
    throwIfFalsy(fn, 'fn');
    this.onActionStarted(name);

    const promises = this.progressive(this.list.map(fn));
    this.list = await Promise.all(promises);

    this.onActionEnded(name);
  }

  async reset(name: string, fn: ResetFn): Promise<void> {
    throwIfFalsy(fn, 'fn');
    this.onActionStarted(name);

    this.list = await fn(this.list);

    this.onActionEnded(name);
  }

  async filter(name: string, fn: FilterFn): Promise<void> {
    throwIfFalsy(fn, 'fn');
    this.onActionStarted(name);

    const progBar = this.progressBar();
    this.list = await filterAsync(this.list, fn, () => progBar.tick());

    this.onActionEnded(name);
  }

  async forEach(name: string, fn: ForEachFn): Promise<void> {
    throwIfFalsy(fn, 'fn');
    this.onActionStarted(name);

    const promises = this.progressive(this.list.map(fn));
    await Promise.all(promises);

    this.onActionEnded(name);
  }

  logList() {
    // tslint:disable-next-line no-console
    console.log(this.list);
  }

  startLogging() {
    this.autoLog = true;
  }

  stopLogging() {
    this.autoLog = false;
  }

  private onActionStarted(name: string) {
    this.logName(name);
    this.startTime = performance.now();
  }

  private onActionEnded(_: string) {
    this.logLengthIfNeeded();
    this.logIfListNeeded();
    const duration = performance.now() - this.startTime;
    this.logTime(prettyMS(duration));
  }

  private logIfListNeeded() {
    if (this.autoLog) {
      this.logList();
    }
  }

  private logName(name: string) {
    throwIfFalsy(name, 'name');
    log(colors.cyan(`🦁 ${name}`));
  }

  private logTime(s: string) {
    log(colors.gray(`> Done in ${s}`));
  }

  private logLengthIfNeeded() {
    if (this.prevLength !== this.list.length) {
      const msg =
        this.prevLength >= 0
          ? `${this.prevLength} >> ${this.list.length}`
          : `${this.list.length}`;
      log(colors.yellow(`> ${msg}`));
      this.prevLength = this.list.length;
    }
  }

  private progressBar(): ProgressBar {
{'fraction_non_alphanumeric': 0.15494791666666666, 'fraction_numerical': 0.0011574074074074073, 'mean_word_length': 3.372549019607843, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5675281', 'n_tokens_mistral': 2370, 'n_tokens_neox': 2319, 'n_words': 509}