<commit_before>'use strict';

const path = require('path');

const beautify = require('js-beautify').html;
const Feplet = require('feplet');
const fs = require('fs-extra');
const he = require('he');
const html2json = require('html2json').html2json;
const json2html = require('html2json').json2html;
const utils = require('fepper-utils');

class HtmlObj {
  constructor() {
    this.node = 'root';
    this.child = [];
  }
}

class JsonForData {
  constructor() {
    this.scrape = [{}];
  }
}

// Exporting module.exports as a class so req and res can be responsibly scoped to the "this" keyword.
module.exports = class {
  constructor(req, res, conf, gatekeeper, html) {
    this.req = req;
    this.res = res;
    this.conf = conf;
    this.gatekeeper = gatekeeper;
    this.html = html;

    // Set some defaults for possibly non-existent nested request properties.
    this.filename = (this.req && this.req.body && this.req.body.filename) || '';
    this.html2json = (this.req && this.req.body && this.req.body.html2json) || '';
    this.json = (this.req && this.req.body && this.req.body.json) || '';
    this.mustache = (this.req && this.req.body && this.req.body.mustache) || '';
    this.selector = (this.req && this.req.body && this.req.body.selector) || '';
    this.url = (this.req && this.req.body && this.req.body.url) || '';
  }

  /**
   * @param {string} selectorRaw - CSS selector plus optional array index.
   * @returns {object} An object storing properties of the selector.
   */
  elementParse(selectorRaw) {
    const selectorObj = this.selectorValidate(selectorRaw);
    const index = selectorObj.index;
    let name;
    let type;

    switch (selectorObj.name[0]) {
      case '#':
        name = selectorObj.name.slice(1);
        type = 'id';
        break;
      case '.':
        name = selectorObj.name.slice(1);
        type = 'class';
        break;
      default:
        name = selectorObj.name;
        type = 'tag';
    }

    return {name, index, type};
  }

  /**
   * Recurse through the object returned by html2json to find the selected element(s).
   *
   * @param {string|object} selector_ - At level 0, a CSS selector. At deeper levels, the selector object.
   * @param {object} html2jsonObj - The object returned by html2json.
   * @param {object|null} persistentObj_ - A mutating object persisting to return results. Not submitted at level 0.
   * @param {number} level - The level of recursion. Not submitted at level 0.
   * @returns {object} An html2json object containing the matched elements. Only returns at level 0.
   */
  elementSelect(selector_, html2jsonObj, persistentObj_ = null, level = 0) {
    // Validate 1st param.
    // Validate 2nd param.
    if (!html2jsonObj || html2jsonObj.constructor !== Object || !Array.isArray(html2jsonObj.child)) {
      return null;
    }

    const persistentObj = persistentObj_ || new HtmlObj();
    let selectorObj;

    if (typeof selector_ === 'string') {
      selectorObj = this.elementParse(selector_);
    }
    else if (selector_ && selector_.constructor === Object) {
      selectorObj = selector_;
    }
    else {
      return null;
    }

    const selectorName = selectorObj.name;
    const selectorType = selectorObj.type;
    persistentObj.index = selectorObj.index;

    for (let i = 0; i < html2jsonObj.child.length; i++) {
      let child = html2jsonObj.child[i];

      if (!child || child.constructor !== Object) {
        continue;
      }

      if (child.node !== 'element') {
        continue;
      }

      // If the element matches selector, push that node onto persistentObj.child.
      let matched = false;
      switch (selectorType) {
        case 'tag':
          if (child.tag && child.tag === selectorName) {
            persistentObj.child.push(child);
            matched = true;
          }
          break;
        case 'id':
          if (child.attr && child.attr.id && child.attr.id === selectorName) {
            persistentObj.child.push(child);
            matched = true;
          }
          break;
        case 'class':
          if (child.attr && child.attr.class && child.attr.class.indexOf(selectorName) > -1) {
            persistentObj.child.push(child);
            matched = true;
          }
          break;
      }

      if (matched) {
        persistentObj.child.push({node: 'text', text: '\n'});
      }

      // Else if recursable, recurse.
      else if (Array.isArray(child.child) && child.child.length) {
        this.elementSelect(selectorObj, child, persistentObj, level + 1);
      }
    }

    if (!level) {
      return persistentObj;
    }
  }

  /**
   * @param {string} scrapeDir - Write destination directory.
   * @param {string} filename - Filename.
   * @param {string} fileMustache - Mustache file's content.
   * @param {string} fileJson - JSON file's content.
   */
  filesWrite(scrapeDir, filename, fileMustache, fileJson) {
    try {
      fs.outputFileSync(scrapeDir + '/' + filename + '.mustache', fileMustache);
      fs.outputFileSync(scrapeDir + '/' + filename + '.json', fileJson);

      let msg = 'Go back to the "Fepper - scrape-html-scraper" tab and refresh the browser to check that your ' +
        'template appears under the "Scrape" menu.';

      this.redirectWithMsg('success', msg, '', '');
    }
    catch (err) {
      utils.error(err);
    }
  }

  /**
   * @param {object} jsonForData - JSON for data ingestion by Mustache. To be stringified for submission to file write.
   * @param {object} targetHtml_ - targeted HTML to be displayed to end-user.
   * @param {string} mustache - Mustache code for submission to file write.
   * @param {string} msgClass - e.g. success, error.
   * @param {string} message - Message to end-user.
   * @returns {string} HTML.
   */
  htmlOutput(jsonForData, targetHtml_, mustache, msgClass = '', message = '') {
    const dataStr = JSON.stringify(jsonForData, null, 2);
    const targetHtml = he.encode(targetHtml_).replace(/\n/g, '<br>');
    const url = this.url;<commit_msg>updated html scraper success message<commit_after>'use strict';

const path = require('path');

const beautify = require('js-beautify').html;
const Feplet = require('feplet');
const fs = require('fs-extra');
const he = require('he');
const html2json = require('html2json').html2json;
const json2html = require('html2json').json2html;
const utils = require('fepper-utils');

class HtmlObj {
  constructor() {
    this.node = 'root';
    this.child = [];
  }
}

class JsonForData {
  constructor() {
    this.scrape = [{}];
  }
}

// Exporting module.exports as a class so req and res can be responsibly scoped to the "this" keyword.
module.exports = class {
  constructor(req, res, conf, gatekeeper, html) {
    this.req = req;
    this.res = res;
    this.conf = conf;
    this.gatekeeper = gatekeeper;
    this.html = html;

    // Set some defaults for possibly non-existent nested request properties.
    this.filename = (this.req && this.req.body && this.req.body.filename) || '';
    this.html2json = (this.req && this.req.body && this.req.body.html2json) || '';
    this.json = (this.req && this.req.body && this.req.body.json) || '';
    this.mustache = (this.req && this.req.body && this.req.body.mustache) || '';
    this.selector = (this.req && this.req.body && this.req.body.selector) || '';
    this.url = (this.req && this.req.body && this.req.body.url) || '';
  }

  /**
   * @param {string} selectorRaw - CSS selector plus optional array index.
   * @returns {object} An object storing properties of the selector.
   */
  elementParse(selectorRaw) {
    const selectorObj = this.selectorValidate(selectorRaw);
    const index = selectorObj.index;
    let name;
    let type;

    switch (selectorObj.name[0]) {
      case '#':
        name = selectorObj.name.slice(1);
        type = 'id';
        break;
      case '.':
        name = selectorObj.name.slice(1);
        type = 'class';
        break;
      default:
        name = selectorObj.name;
        type = 'tag';
    }

    return {name, index, type};
  }

  /**
   * Recurse through the object returned by html2json to find the selected element(s).
   *
   * @param {string|object} selector_ - At level 0, a CSS selector. At deeper levels, the selector object.
   * @param {object} html2jsonObj - The object returned by html2json.
   * @param {object|null} persistentObj_ - A mutating object persisting to return results. Not submitted at level 0.
   * @param {number} level - The level of recursion. Not submitted at level 0.
   * @returns {object} An html2json object containing the matched elements. Only returns at level 0.
   */
  elementSelect(selector_, html2jsonObj, persistentObj_ = null, level = 0) {
    // Validate 1st param.
    // Validate 2nd param.
    if (!html2jsonObj || html2jsonObj.constructor !== Object || !Array.isArray(html2jsonObj.child)) {
      return null;
    }

    const persistentObj = persistentObj_ || new HtmlObj();
    let selectorObj;

    if (typeof selector_ === 'string') {
      selectorObj = this.elementParse(selector_);
    }
    else if (selector_ && selector_.constructor === Object) {
      selectorObj = selector_;
    }
    else {
      return null;
    }

    const selectorName = selectorObj.name;
    const selectorType = selectorObj.type;
    persistentObj.index = selectorObj.index;

    for (let i = 0; i < html2jsonObj.child.length; i++) {
      let child = html2jsonObj.child[i];

      if (!child || child.constructor !== Object) {
        continue;
      }

      if (child.node !== 'element') {
        continue;
      }

      // If the element matches selector, push that node onto persistentObj.child.
      let matched = false;
      switch (selectorType) {
        case 'tag':
          if (child.tag && child.tag === selectorName) {
            persistentObj.child.push(child);
            matched = true;
          }
          break;
        case 'id':
          if (child.attr && child.attr.id && child.attr.id === selectorName) {
            persistentObj.child.push(child);
            matched = true;
          }
          break;
        case 'class':
          if (child.attr && child.attr.class && child.attr.class.indexOf(selectorName) > -1) {
            persistentObj.child.push(child);
            matched = true;
          }
          break;
      }

      if (matched) {
        persistentObj.child.push({node: 'text', text: '\n'});
      }

      // Else if recursable, recurse.
      else if (Array.isArray(child.child) && child.child.length) {
        this.elementSelect(selectorObj, child, persistentObj, level + 1);
      }
    }

    if (!level) {
      return persistentObj;
    }
  }

  /**
   * @param {string} scrapeDir - Write destination directory.
   * @param {string} filename - Filename.
   * @param {string} fileMustache - Mustache file's content.
   * @param {string} fileJson - JSON file's content.
   */
  filesWrite(scrapeDir, filename, fileMustache, fileJson) {
    try {
      fs.outputFileSync(scrapeDir + '/' + filename + '.mustache', fileMustache);
      fs.outputFileSync(scrapeDir + '/' + filename + '.json', fileJson);

      let msg = 'Refresh the browser to check that your template appears under the "Scrape" menu.';

      this.redirectWithMsg('success', msg, '', '');
    }
    catch (err) {
      utils.error(err);
    }
  }

  /**
   * @param {object} jsonForData - JSON for data ingestion by Mustache. To be stringified for submission to file write.
   * @param {object} targetHtml_ - targeted HTML to be displayed to end-user.
   * @param {string} mustache - Mustache code for submission to file write.
   * @param {string} msgClass - e.g. success, error.
   * @param {string} message - Message to end-user.
   * @returns {string} HTML.
   */
  htmlOutput(jsonForData, targetHtml_, mustache, msgClass = '', message = '') {
    const dataStr = JSON.stringify(jsonForData, null, 2);
    const targetHtml = he.encode(targetHtml_).replace(/\n/g, '<br>');
    const url = this.url;
{'fraction_non_alphanumeric': 0.1350584307178631, 'fraction_numerical': 0.005342237061769616, 'mean_word_length': 2.57748581666169, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1188307', 'n_tokens_mistral': 3656, 'n_tokens_neox': 3584, 'n_words': 999}