<commit_before>import {EventEmitter} from 'events';
import {ChildProcess, spawn} from "child_process";
import {FFMpegError, FFMpegOutOfMemoryError} from "./error";
import {humanTimeToMS, Parse, pidToResourceUsage} from "./helper";
import ProcessEnv = NodeJS.ProcessEnv;
import Timeout = NodeJS.Timeout;

export * from "./error"

export interface FFMpegProgressOptions {
  cmd?: string
  cwd?: string
  env?: ProcessEnv
  duration?: number
  hideFFConfig?: boolean
  maxMemory?: number
}

export interface IFFMpegFileDetails {
  duration?: number
  bitrate?: number
  start?: number
  resolution?: { width: number, height: number }
  fps?: number
}

export interface IFFMpegProgressData {
  speed?: number
  eta?: number
  time?: number
  progress?: number
  drop?: number
  dup?: number
  fps?: number
  frame?: number
  q?: number
  size?: number
  bitrate?: string

  [s: string]: string | number
}

export interface IFFMpegProgress {
  on(event: 'end', listener: (code: number | undefined, signal: string | undefined) => void): this;

  on(event: 'details', listener: (file: IFFMpegFileDetails) => void): this;

  on(event: 'progress', listener: (p: IFFMpegProgressData) => void): this;

  on(event: 'raw', listener: (text: string) => void): this;
}

export class FFMpegProgress extends EventEmitter implements IFFMpegProgress {

  private _args: string[];
  private _process: ChildProcess;

  private _details: {
    file?: IFFMpegFileDetails
  } = {};
  private _metadataDuration: number = null;
  private _output: string = '';
  private _stderr: string = '';
  private _stdout: string = '';
  private _isKilledByUser: string | false = false;
  private _outOfMemory: boolean = false;
  private _vitalsTimer: Timeout;

  public readonly options: FFMpegProgressOptions;


  constructor(args: string[], options: FFMpegProgressOptions = {}) {
    super();

    this.options = {
      cmd: options.cmd || 'ffmpeg',
      cwd: options.cwd || process.cwd(),
      env: options.env || process.env,
      hideFFConfig: options.hideFFConfig || false,
      maxMemory: Math.max(0, options.maxMemory) || Infinity
    };

    const extra_args = [];
    if (this.options.hideFFConfig) {
      extra_args.push(`-hide_banner`)
    }

    this._args = args.slice();
    this._process = spawn(
      this.options.cmd,
      extra_args.concat(args),
      {
        cwd: this.options.cwd,
        env: this.options.env
      }
    );

    this._process.stdout.on('data', this.processOutput);
    this._process.stderr.on('data', this.processOutput);

    this._process.stdout.on('data', (d: Buffer) => this._stdout += d.toString());
    this._process.stderr.on('data', (d: Buffer) => this._stderr += d.toString());

    this._process.once('close', (code, signal) => {
      this.emit('end', code, signal);
      clearInterval(this._vitalsTimer);
    });

    this._vitalsTimer = setInterval(this._checkVitals.bind(this), 500);
  }

  private async _checkVitals() {
    const vitals = await pidToResourceUsage(this._process.pid);
    if (vitals.memory > this.options.maxMemory) {
      this._outOfMemory = true;
      this.kill();
    }
  }

  kill(signal: string = 'SIGKILL') {
    this._isKilledByUser = signal;
    this._process.kill(signal);
  }

  stop() {
    return this.kill('SIGINT');
  }

  get details(): IFFMpegFileDetails {
    return this._details.file;
  }

  get output(): string {
    return this._output;
  }

  get stderrOutput(): string {
    return this._stderr;
  }

  get stdoutOutput(): string {
    return this._stdout;<commit_msg>Add: soft fail for vitals check<commit_after>import {EventEmitter} from 'events';
import {ChildProcess, spawn} from "child_process";
import {FFMpegError, FFMpegOutOfMemoryError} from "./error";
import {humanTimeToMS, Parse, pidToResourceUsage} from "./helper";
import ProcessEnv = NodeJS.ProcessEnv;
import Timeout = NodeJS.Timeout;

export * from "./error"

export interface FFMpegProgressOptions {
  cmd?: string
  cwd?: string
  env?: ProcessEnv
  duration?: number
  hideFFConfig?: boolean
  maxMemory?: number
}

export interface IFFMpegFileDetails {
  duration?: number
  bitrate?: number
  start?: number
  resolution?: { width: number, height: number }
  fps?: number
}

export interface IFFMpegProgressData {
  speed?: number
  eta?: number
  time?: number
  progress?: number
  drop?: number
  dup?: number
  fps?: number
  frame?: number
  q?: number
  size?: number
  bitrate?: string

  [s: string]: string | number
}

export interface IFFMpegProgress {
  on(event: 'end', listener: (code: number | undefined, signal: string | undefined) => void): this;

  on(event: 'details', listener: (file: IFFMpegFileDetails) => void): this;

  on(event: 'progress', listener: (p: IFFMpegProgressData) => void): this;

  on(event: 'raw', listener: (text: string) => void): this;
}

export class FFMpegProgress extends EventEmitter implements IFFMpegProgress {

  private _args: string[];
  private _process: ChildProcess;

  private _details: {
    file?: IFFMpegFileDetails
  } = {};
  private _metadataDuration: number = null;
  private _output: string = '';
  private _stderr: string = '';
  private _stdout: string = '';
  private _isKilledByUser: string | false = false;
  private _outOfMemory: boolean = false;
  private _vitalsTimer: Timeout;

  public readonly options: FFMpegProgressOptions;


  constructor(args: string[], options: FFMpegProgressOptions = {}) {
    super();

    this.options = {
      cmd: options.cmd || 'ffmpeg',
      cwd: options.cwd || process.cwd(),
      env: options.env || process.env,
      hideFFConfig: options.hideFFConfig || false,
      maxMemory: Math.max(0, options.maxMemory) || Infinity
    };

    const extra_args = [];
    if (this.options.hideFFConfig) {
      extra_args.push(`-hide_banner`)
    }

    this._args = args.slice();
    this._process = spawn(
      this.options.cmd,
      extra_args.concat(args),
      {
        cwd: this.options.cwd,
        env: this.options.env
      }
    );

    this._process.stdout.on('data', this.processOutput);
    this._process.stderr.on('data', this.processOutput);

    this._process.stdout.on('data', (d: Buffer) => this._stdout += d.toString());
    this._process.stderr.on('data', (d: Buffer) => this._stderr += d.toString());

    this._process.once('close', (code, signal) => {
      this.emit('end', code, signal);
      clearInterval(this._vitalsTimer);
    });

    this._vitalsTimer = setInterval(this._checkVitals.bind(this), 500);
  }

  private async _checkVitals() {
    try {
      const vitals = await pidToResourceUsage(this._process.pid);
      if (vitals.memory > this.options.maxMemory) {
        this._outOfMemory = true;
        this.kill();
      }
    } catch (e) {
      if (!e.stack) {
        Error.captureStackTrace(e);
      }
      console.error(`Vitals check for PID:${this._process.pid} resulted in: ${e.stack}`);
    }
  }

  kill(signal: string = 'SIGKILL') {
    this._isKilledByUser = signal;
    this._process.kill(signal);
  }

  stop() {
    return this.kill('SIGINT');
  }

  get details(): IFFMpegFileDetails {
    return this._details.file;
  }

  get output(): string {
    return this._output;
  }

  get stderrOutput(): string {
    return this._stderr;
  }

  get stdoutOutput(): string {
    return this._stdout;
{'fraction_non_alphanumeric': 0.13674979446423677, 'fraction_numerical': 0.0010961907371882709, 'mean_word_length': 3.4916923076923077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1620221', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2278, 'n_words': 563}