<commit_before>import Queue from './queue';
import COFetch from './cofetch';
import * as EventEmitter from 'events';

interface Gallery {
  gid: string;
  token: string;
}

export interface Page {
  filename?: string;
  imgsrc?: string;
  failnl?: Set<string>;
  style?: string;
  url?: string;
  imgkey?: string;
  page?: number;
}

interface Headers {
  [key: string]: string;
}

export class EhRetriever extends EventEmitter {
  url: string;
  html: string;
  gallery: Gallery;
  referer: string;
  showkey: string;
  ehentaiHost: string;
  pages: Promise<Page[]>;
  q: Queue<Response>;
  onPageLoadCallback: ((step: number, all: number) => void)[];
  promiseInit: Promise<void>;

  constructor(url: string, html: string) {
    super();

    const testEXHentaiUrl = /^https?:\/\/(?:e-|ex)hentai\.org\//;

    if (typeof url !== 'string') {
      throw new TypeError('invalid `url`, expected a string');
    }

    if (!testEXHentaiUrl.test(url)) {
      throw new TypeError(`invalid url: ${url}`);
    }

    this.url = url;
    this.html = html;
    this.gallery = { gid: '', token: '' };
    this.referer = url;
    this.showkey = '';
    this.ehentaiHost = testEXHentaiUrl.exec(url)[0].slice(0, -1);
    this.q = new Queue(3, 3000, 1000);
    this.pages = this.init();
    this.pages.then(() => this.emit('ready'));
  }

  private async init(): Promise<Page[]> {
      if (!this.html) {
        this.html = await this.fetch(this.url).then(res => res.text());
      }<commit_msg>remove old properties that are not used anymore<commit_after>import Queue from './queue';
import COFetch from './cofetch';
import * as EventEmitter from 'events';

interface Gallery {
  gid: string;
  token: string;
}

export interface Page {
  filename?: string;
  imgsrc?: string;
  failnl?: Set<string>;
  style?: string;
  url?: string;
  imgkey?: string;
  page?: number;
}

interface Headers {
  [key: string]: string;
}

export class EhRetriever extends EventEmitter {
  url: string;
  html: string;
  gallery: Gallery;
  referer: string;
  showkey: string;
  ehentaiHost: string;
  q: Queue<Response>;
  pages: Promise<Page[]>;

  constructor(url: string, html: string) {
    super();

    const testEXHentaiUrl = /^https?:\/\/(?:e-|ex)hentai\.org\//;

    if (typeof url !== 'string') {
      throw new TypeError('invalid `url`, expected a string');
    }

    if (!testEXHentaiUrl.test(url)) {
      throw new TypeError(`invalid url: ${url}`);
    }

    this.url = url;
    this.html = html;
    this.gallery = { gid: '', token: '' };
    this.referer = url;
    this.showkey = '';
    this.ehentaiHost = testEXHentaiUrl.exec(url)[0].slice(0, -1);
    this.q = new Queue(3, 3000, 1000);
    this.pages = this.init();
    this.pages.then(() => this.emit('ready'));
  }

  private async init(): Promise<Page[]> {
      if (!this.html) {
        this.html = await this.fetch(this.url).then(res => res.text());
      }
{'fraction_non_alphanumeric': 0.1694740460639395, 'fraction_numerical': 0.008250257820556892, 'mean_word_length': 3.211287988422576, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3664685', 'n_tokens_mistral': 1023, 'n_tokens_neox': 1002, 'n_words': 255}