<commit_before>const P = require('bluebird');

const DocumentWorker = require('../html/DocumentWorker');
const thumbnail = require('../thumbnail');
const NodeType = require('../nodeType');
const domUtil = require('../domUtil');
const pagelib = require('../../pagelib/build/wikimedia-page-library-transform');
const Edit = pagelib.EditTransform;
const WidenImage = pagelib.WidenImage;
const LazyLoad = pagelib.LazyLoadTransform;
const Table = pagelib.CollapseTable;
const LeadIntroduction = pagelib.LeadIntroductionTransform;
const constants = require('./MobileHTMLConstants');
const head = require('../transformations/pcs/head');
const addPageHeader = require('../transformations/pcs/addPageHeader');
const addDescriptionSourceLeadSection = require('../transformations/pcs/addDescriptionSourceLeadSection');
const parseProperty = require('../parseProperty');

/**
 * MobileHTML is the prepareor for mobile html output.
 * It handles the expensive transforms that need to be
 * applied to a Parsoid document in preparation for mobile display,
 * @param {!Document} doc Parsoid document to process
 */
class MobileHTML extends DocumentWorker {
/**
 * prepareElement receives every element as it is iterated
 * over and performs the required transforms. The DOM should
 * not be manipulated inside of this method. Instead, save
 * items for manipulation and perform the manipulation in
 * finalize
 * @param {!Element} element element to process
 */
  prepareElement(element) {
    const id = element.getAttribute('id');
    const tagName = element.tagName;
    const cls = element.getAttribute('class');
    if (this.isRemovableElement(element, tagName, id, cls)) {
      // save here to manipulate the dom later
      this.markForRemoval(element);
    } else if (this.isReference(element)) {
      // save here to manipulate the dom later
      this.referenceElements.push(element);
    } else {
      switch (tagName) {
        case 'A':
          this.prepareAnchor(element, cls);
          break;
        case 'LINK':
          this.makeSchemeless(element, 'href');
          break;
        case 'SCRIPT':
        case 'SOURCE':
          this.makeSchemeless(element, 'src');
          break;
        case 'SECTION':
          this.currentSectionId = element.getAttribute('data-mw-section-id');
          // save for later due to DOM manipulation
          this.sections[this.currentSectionId] = element;
          break;
        case 'IMG':
          this.prepareImage(element);
          break;
        case 'DIV':
          this.prepareDiv(element, cls);
          break;
        case 'TABLE':
          // Images in tables should not be widened
          this.widenImageExcludedNode = element;
          this.prepareTable(element, cls);
          break;
        default:
          if (!this.headers[this.currentSectionId] && constants.headerTagRegex.test(tagName)) {
            this.headers[this.currentSectionId] = element;
          }
          break;
      }

      if (!this.widenImageExcludedNode && constants.widenImageExclusionClassRegex.test(cls)) {
        this.widenImageExcludedNode = element;
      }

      if (this.themeExcludedNode) {
        element.classList.add('notheme');
      } else {
        const style = element.getAttribute('style');
        if (style && constants.inlineBackgroundStyleRegex.test(style)) {
          this.themeExcludedNode = element;
          element.classList.add('notheme');
        }
      }

      const attributesToRemove = constants.attributesToRemoveFromElements[tagName];
      if (attributesToRemove) {
        for (const attrib of attributesToRemove) {
          element.removeAttribute(attrib);
        }
      }

      if (id && constants.mwidRegex.test(id)) {
        element.removeAttribute('id');
      }
    }
  }

  /**
   * Run the next finalization step. All of the DOM manipulation occurs here because
   * manipulating the DOM while walking it will result in an incomplete walk.
   */
  finalizeStep() {
    let node;

    node = this.nodesToRemove.pop();
    if (node) {
      const ancestor = node.parentNode;
      if (ancestor) {
        ancestor.removeChild(node);
      }
      return true;
    }

    node = this.referenceElements.pop();
    if (node) {
      this.prepareReference(node, this.doc);
      return true;
    }

    node = this.infoboxes.pop();
    if (node) {
      this.prepareInfobox(node);
      return true;
    }

    node = this.redLinks.pop();
    if (node) {
      this.prepareRedLink(node, this.doc);
      return true;
    }

    node = this.lazyLoadableImages.pop();
    if (node) {
      LazyLoad.convertImageToPlaceholder(this.doc, node);
      return true;
    }

    if (!this.sectionIds) {
      this.sectionIds = Object.keys(this.sections);
    }

    const sectionId = this.sectionIds.pop();
    if (sectionId) {
      this.prepareSection(sectionId);
      return true;
    }

    const pcs = this.doc.createElement('div');
    pcs.setAttribute('id', 'pcs');
    const body = this.doc.body;
    const children = Array.from(body.children);
    for (const child of children) {
      pcs.appendChild(child);
    }
    body.appendChild(pcs);

    head.addCssLinks(this.doc, this.metadata);
    head.addMetaViewport(this.doc);
    head.addPageLibJs(this.doc, this.metadata);

    return false;
  }

/**
 * Returns a MobileHTML object ready for processing
 * @param {!Document} doc document to process
 * @param {?Object} metadata metadata object that should include:
 *   {!string} baseURI the baseURI for the REST API
 *   {!string} revision the revision of the page
 *   {!string} tid the tid of the page
 */
  constructor(doc, metadata) {
    super(doc);
    this.prepareDoc(doc);
    this.nodesToRemove = [];
    this.referenceElements = [];
    this.lazyLoadableImages = [];
    this.redLinks = [];
    this.infoboxes = [];
    this.headers = {};
    this.sections = {};
    this.currentSectionId = 0;
    this.metadata = metadata || {};
    this.metadata.pronunciation = parseProperty.parsePronunciation(doc);
    this.metadata.linkTitle = domUtil.getParsoidLinkTitle(doc);
    this.metadata.plainTitle = domUtil.getParsoidPlainTitle(doc);
  }

/**
 * Adds metadata to the resulting document
 * @param {?Object} mw metadata from MediaWiki
 */<commit_msg>Hygiene: Add expected properties to function doc<commit_after>const P = require('bluebird');

const DocumentWorker = require('../html/DocumentWorker');
const thumbnail = require('../thumbnail');
const NodeType = require('../nodeType');
const domUtil = require('../domUtil');
const pagelib = require('../../pagelib/build/wikimedia-page-library-transform');
const Edit = pagelib.EditTransform;
const WidenImage = pagelib.WidenImage;
const LazyLoad = pagelib.LazyLoadTransform;
const Table = pagelib.CollapseTable;
const LeadIntroduction = pagelib.LeadIntroductionTransform;
const constants = require('./MobileHTMLConstants');
const head = require('../transformations/pcs/head');
const addPageHeader = require('../transformations/pcs/addPageHeader');
const addDescriptionSourceLeadSection = require('../transformations/pcs/addDescriptionSourceLeadSection');
const parseProperty = require('../parseProperty');

/**
 * MobileHTML is the prepareor for mobile html output.
 * It handles the expensive transforms that need to be
 * applied to a Parsoid document in preparation for mobile display,
 * @param {!Document} doc Parsoid document to process
 */
class MobileHTML extends DocumentWorker {
/**
 * prepareElement receives every element as it is iterated
 * over and performs the required transforms. The DOM should
 * not be manipulated inside of this method. Instead, save
 * items for manipulation and perform the manipulation in
 * finalize
 * @param {!Element} element element to process
 */
  prepareElement(element) {
    const id = element.getAttribute('id');
    const tagName = element.tagName;
    const cls = element.getAttribute('class');
    if (this.isRemovableElement(element, tagName, id, cls)) {
      // save here to manipulate the dom later
      this.markForRemoval(element);
    } else if (this.isReference(element)) {
      // save here to manipulate the dom later
      this.referenceElements.push(element);
    } else {
      switch (tagName) {
        case 'A':
          this.prepareAnchor(element, cls);
          break;
        case 'LINK':
          this.makeSchemeless(element, 'href');
          break;
        case 'SCRIPT':
        case 'SOURCE':
          this.makeSchemeless(element, 'src');
          break;
        case 'SECTION':
          this.currentSectionId = element.getAttribute('data-mw-section-id');
          // save for later due to DOM manipulation
          this.sections[this.currentSectionId] = element;
          break;
        case 'IMG':
          this.prepareImage(element);
          break;
        case 'DIV':
          this.prepareDiv(element, cls);
          break;
        case 'TABLE':
          // Images in tables should not be widened
          this.widenImageExcludedNode = element;
          this.prepareTable(element, cls);
          break;
        default:
          if (!this.headers[this.currentSectionId] && constants.headerTagRegex.test(tagName)) {
            this.headers[this.currentSectionId] = element;
          }
          break;
      }

      if (!this.widenImageExcludedNode && constants.widenImageExclusionClassRegex.test(cls)) {
        this.widenImageExcludedNode = element;
      }

      if (this.themeExcludedNode) {
        element.classList.add('notheme');
      } else {
        const style = element.getAttribute('style');
        if (style && constants.inlineBackgroundStyleRegex.test(style)) {
          this.themeExcludedNode = element;
          element.classList.add('notheme');
        }
      }

      const attributesToRemove = constants.attributesToRemoveFromElements[tagName];
      if (attributesToRemove) {
        for (const attrib of attributesToRemove) {
          element.removeAttribute(attrib);
        }
      }

      if (id && constants.mwidRegex.test(id)) {
        element.removeAttribute('id');
      }
    }
  }

  /**
   * Run the next finalization step. All of the DOM manipulation occurs here because
   * manipulating the DOM while walking it will result in an incomplete walk.
   */
  finalizeStep() {
    let node;

    node = this.nodesToRemove.pop();
    if (node) {
      const ancestor = node.parentNode;
      if (ancestor) {
        ancestor.removeChild(node);
      }
      return true;
    }

    node = this.referenceElements.pop();
    if (node) {
      this.prepareReference(node, this.doc);
      return true;
    }

    node = this.infoboxes.pop();
    if (node) {
      this.prepareInfobox(node);
      return true;
    }

    node = this.redLinks.pop();
    if (node) {
      this.prepareRedLink(node, this.doc);
      return true;
    }

    node = this.lazyLoadableImages.pop();
    if (node) {
      LazyLoad.convertImageToPlaceholder(this.doc, node);
      return true;
    }

    if (!this.sectionIds) {
      this.sectionIds = Object.keys(this.sections);
    }

    const sectionId = this.sectionIds.pop();
    if (sectionId) {
      this.prepareSection(sectionId);
      return true;
    }

    const pcs = this.doc.createElement('div');
    pcs.setAttribute('id', 'pcs');
    const body = this.doc.body;
    const children = Array.from(body.children);
    for (const child of children) {
      pcs.appendChild(child);
    }
    body.appendChild(pcs);

    head.addCssLinks(this.doc, this.metadata);
    head.addMetaViewport(this.doc);
    head.addPageLibJs(this.doc, this.metadata);

    return false;
  }

/**
 * Returns a MobileHTML object ready for processing
 * @param {!Document} doc document to process
 * @param {?Object} metadata metadata object that should include:
 *   {!string} baseURI the baseURI for the REST API
 *   {!string} revision the revision of the page
 *   {!string} tid the tid of the page
 */
  constructor(doc, metadata) {
    super(doc);
    this.prepareDoc(doc);
    this.nodesToRemove = [];
    this.referenceElements = [];
    this.lazyLoadableImages = [];
    this.redLinks = [];
    this.infoboxes = [];
    this.headers = {};
    this.sections = {};
    this.currentSectionId = 0;
    this.metadata = metadata || {};
    this.metadata.pronunciation = parseProperty.parsePronunciation(doc);
    this.metadata.linkTitle = domUtil.getParsoidLinkTitle(doc);
    this.metadata.plainTitle = domUtil.getParsoidPlainTitle(doc);
  }

/**
 * Adds metadata to the resulting document
 * @param {?Object} mw metadata from MediaWiki with:
 *   {!array} protection
 *   {!string} displaytitle
 *   {?string} description
 *   {?string} description_source
 */
{'fraction_non_alphanumeric': 0.11586663513833058, 'fraction_numerical': 0.0001576416804603137, 'mean_word_length': 2.837870538415003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3709408', 'n_tokens_mistral': 3621, 'n_tokens_neox': 3571, 'n_words': 899}