<commit_before>import Queue from './queue';
import COFetch from './cofetch';

class ehRetriever {
  constructor(url, html) {
    if (typeof url !== 'string') throw new TypeError('invalid `url`, expected a string');
    if (url.search(/^https?:\/\//) < 0) throw new TypeError(`invalid url: ${url}`);

    this.url = url;
    this.html = html;
    this.gallery = {
      'gid':   undefined,
      'token': undefined
    }
    this.referer = url;
    this.showkey = undefined;
    this.exprefix = url.search(/exhentai/) >= 0 ? 'ex' : 'g.e-';
    this.pages = [];
    this.q = new Queue(3, { 'timeout': 3000, 'delay': 1000 });
    this.onPageLoadCallback = [];

    this.promiseInit = this.init();
  }

  async init() {
      if (!this.html) this.html = await this.fetch(this.url).then(res => res.text());

      const galleryURL = this.html.match(/hentai\.org\/g\/(\d+)\/([a-z0-9]+)/i);
      const showkey = this.html.match(/showkey="([^"]+)"/i);

      if (galleryURL) {
        this.gallery.gid = galleryURL[1];
        this.gallery.token = galleryURL[2];
      }
      else {
        throw new Error("Can't get gallery URL");
      }

      if (showkey) this.showkey = showkey[1];
      else throw new Error("Can't get showkey");

      this.pages = await this.getAllPageURL();
  }

  async getAllPageURL() {
    const firstPage = await this.fetch(`http://${this.exprefix}hentai.org/g/${this.gallery.gid}/${this.gallery.token}`).then(res => res.text());

    let pageNum = firstPage.match(/<table[^>]*class="ptt"[^>]*>((?:[^<]*)(?:<(?!\/table>)[^<]*)*)<\/table>/);
    if (pageNum) pageNum = pageNum[1].match(/g\/[^/]+\/[^/]+\/\?p=\d+$/g);
    else throw new Error('Cant get page numbers');

    // A gallery containing only one page won't have page number in gallery link.
    // So, if pageNum is null, the gallery has only one page.
    if (pageNum) {
      pageNum = pageNum.map(e => parseInt(e.replace(/.*(\d+)$/, '$1'), 10));
    }
    const pageNumMax = (pageNum ? Math.max.apply(null, pageNum) : 0);

    const allPages = await Promise.all(
      Array(pageNumMax).fill().map( (e, i) => {
        return this.fetch(`http://${this.exprefix}hentai.org/g/${this.gallery.gid}/${this.gallery.token}/?p=${i + 1}`).then(res => res.text());
      })
    );
    allPages.unshift(firstPage);

    return allPages
      .map(e => e.match(/<div[^>]*class="gdt[lm]"[^>]*>(?:(?:[^<]*)(?:<(?!\/div>)[^<]*)*)<\/div>/g))
      .reduce( (p, c) => p.concat(c) )
      .map(e => {
        let tokens = e.match(/s\/[a-z0-9]+\/\d+-\d+/)[0].split('/');
        return {
          'imgkey': tokens[1],
          'page': parseInt(tokens[2].split('-')[1], 10)<commit_msg>fix #3<commit_after>import Queue from './queue';
import COFetch from './cofetch';

class ehRetriever {
  constructor(url, html) {
    if (typeof url !== 'string') throw new TypeError('invalid `url`, expected a string');
    if (url.search(/^https?:\/\//) < 0) throw new TypeError(`invalid url: ${url}`);

    this.url = url;
    this.html = html;
    this.gallery = {
      'gid':   undefined,
      'token': undefined
    }
    this.referer = url;
    this.showkey = undefined;
    this.exprefix = url.search(/exhentai/) >= 0 ? 'ex' : 'g.e-';
    this.pages = [];
    this.q = new Queue(3, { 'timeout': 3000, 'delay': 1000 });
    this.onPageLoadCallback = [];

    this.promiseInit = this.init();
  }

  async init() {
      if (!this.html) this.html = await this.fetch(this.url).then(res => res.text());

      const galleryURL = this.html.match(/hentai\.org\/g\/(\d+)\/([a-z0-9]+)/i);
      const showkey = this.html.match(/showkey="([^"]+)"/i);

      if (galleryURL) {
        this.gallery.gid = galleryURL[1];
        this.gallery.token = galleryURL[2];
      }
      else {
        throw new Error("Can't get gallery URL");
      }

      if (showkey) this.showkey = showkey[1];
      else throw new Error("Can't get showkey");

      this.pages = await this.getAllPageURL();
  }

  async getAllPageURL() {
    const firstPage = await this.fetch(`http://${this.exprefix}hentai.org/g/${this.gallery.gid}/${this.gallery.token}`).then(res => res.text());

    let pageNum = firstPage.match(/<table[^>]*class="ptt"[^>]*>((?:[^<]*)(?:<(?!\/table>)[^<]*)*)<\/table>/);
    if (pageNum) pageNum = pageNum[1].match(/g\/[^/]+\/[^/]+\/\?p=\d+/g);
    else throw new Error('Cant get page numbers');

    // A gallery containing only one page won't have page number in gallery link.
    // So, if pageNum is null, the gallery has only one page.
    if (pageNum) {
      pageNum = pageNum.map(e => parseInt(e.match(/(\d+)$/)[1], 10));
    }
    const pageNumMax = (pageNum ? Math.max.apply(null, pageNum) : 0);

    const allPages = await Promise.all(
      Array(pageNumMax).fill().map( (e, i) => {
        return this.fetch(`http://${this.exprefix}hentai.org/g/${this.gallery.gid}/${this.gallery.token}/?p=${i + 1}`).then(res => res.text());
      })
    );
    allPages.unshift(firstPage);

    return allPages
      .map(e => e.match(/<div[^>]*class="gdt[lm]"[^>]*>(?:(?:[^<]*)(?:<(?!\/div>)[^<]*)*)<\/div>/g))
      .reduce( (p, c) => p.concat(c) )
      .map(e => {
        let tokens = e.match(/s\/[a-z0-9]+\/\d+-\d+/)[0].split('/');
        return {
          'imgkey': tokens[1],
          'page': parseInt(tokens[2].split('-')[1], 10)
{'fraction_non_alphanumeric': 0.233138106933686, 'fraction_numerical': 0.01152465520498772, 'mean_word_length': 2.722925457102672, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '306017', 'n_tokens_mistral': 1989, 'n_tokens_neox': 1899, 'n_words': 375}