<commit_before>'use strict';

/*
    See https://phabricator.wikimedia.org/T170690
    Turns reference list content into a JSON structure.
*/

const _ = require('underscore');
const NodeType = require('../../nodeType');
const CITATION_TYPES = [ 'web', 'news', 'journal', 'book' ];
const DEFAULT_CITATION_TYPE = 'generic';

const hasOnlyWhitespace = node =>
    node.nodeType === NodeType.TEXT_NODE && /^\s*$/.test(node.textContent);

const getCiteNoteId = (listItemElement) => listItemElement.getAttribute('id');

/**
 * @return {!Object} of {href, text}.
 */
const buildBackLinkObject = (element) => {
    return {
        href: element.getAttribute('href'),
        text: _.escape(element.textContent.trim())
    };
};

/**
 * Parses multiple back links wrapped in a span element.
 * @param {!Element} element 'span.mw-reference-text'
 * @return {!Object} of {href, text}.
 */
const structureMultipleBackLinks = (element) => {
    const resultArray = [];

    // TODO: check with domino guys if we should use a different API here.
    // https://github.com/fgnass/domino/blob/master/CHANGELOG.md
    const spanChildElements = element.children;
    for (let i = 0; i < spanChildElements.length; i++) {
        const spanChildEl = spanChildElements[i];
        if (spanChildEl.tagName === 'A') {
            resultArray.push(buildBackLinkObject(spanChildEl));
        }
    }
    return resultArray;
};

const structureBackLinks = (listItemElement, logger) => {
    let backLinks = [];
    const singleBackLinksElement = listItemElement.querySelector('a[rel=mw:referencedBy]');
    if (singleBackLinksElement) {
        backLinks = [ buildBackLinkObject(singleBackLinksElement) ];
    } else {
        const multipleBackLinksRoot = listItemElement.querySelector('span[rel=mw:referencedBy]')
            || listItemElement.querySelector('span.mw-cite-backlink');
        if (multipleBackLinksRoot) {
            backLinks = structureMultipleBackLinks(multipleBackLinksRoot);
        } else {
            logger.log('warn/ref', `no back links found for ${getCiteNoteId(listItemElement)}`);
        }
    }
    return backLinks;
};

const collectCitationsOfOneCiteElement = (citations, citeElement) => {
    const classes = citeElement.classList;
    for (let k = 0; k < classes.length; k++) {
        if (classes[k] !== 'citation') {
            citations.add(classes[k]);
        }
    }
};

const addCitationsToContent = (citations, searchElement) => {
    if (searchElement.tagName === 'CITE') {
        collectCitationsOfOneCiteElement(citations, searchElement);
    } else {
        const citeElements = searchElement.querySelectorAll('cite');
        for (let i = 0; i < citeElements.length; i++) {<commit_msg>Dev: Drop "no back links" logging to the info level<commit_after>'use strict';

const _ = require('underscore');
const NodeType = require('../../nodeType');
const CITATION_TYPES = [ 'web', 'news', 'journal', 'book' ];
const DEFAULT_CITATION_TYPE = 'generic';
const REGEX_WHITESPACE = /^\s*$/;

const hasOnlyWhitespace = node =>
    node.nodeType === NodeType.TEXT_NODE && REGEX_WHITESPACE.test(node.textContent);

const getCiteNoteId = (listItemElement) => listItemElement.getAttribute('id');

/**
 * @return {!Object} of {href, text}.
 */
const buildBackLinkObject = (element) => {
    return {
        href: element.getAttribute('href'),
        text: _.escape(element.textContent.trim())
    };
};

/**
 * Parses multiple back links wrapped in a span element.
 * @param {!Element} element 'span.mw-reference-text'
 * @return {!Object} of {href, text}.
 */
const structureMultipleBackLinks = (element) => {
    const resultArray = [];

    // TODO: check with domino guys if we should use a different API here.
    // https://github.com/fgnass/domino/blob/master/CHANGELOG.md
    const spanChildElements = element.children;
    for (let i = 0; i < spanChildElements.length; i++) {
        const spanChildEl = spanChildElements[i];
        if (spanChildEl.tagName === 'A') {
            resultArray.push(buildBackLinkObject(spanChildEl));
        }
    }
    return resultArray;
};

/**
 * @param {!Element} listItemElement
 * @param {!Logger} logger
 * @returns {[]}
 */
const structureBackLinks = (listItemElement, logger) => {
    let backLinks = [];
    const singleBackLinksElement = listItemElement.querySelector('a[rel=mw:referencedBy]');
    if (singleBackLinksElement) {
        backLinks = [ buildBackLinkObject(singleBackLinksElement) ];
    } else {
        const multipleBackLinksRoot = listItemElement.querySelector('span[rel=mw:referencedBy]')
            || listItemElement.querySelector('span.mw-cite-backlink');
        if (multipleBackLinksRoot) {
            backLinks = structureMultipleBackLinks(multipleBackLinksRoot);
        } else {
            logger.log('info/ref', `no back links found for ${getCiteNoteId(listItemElement)}`);
        }
    }
    return backLinks;
};

const collectCitationsOfOneCiteElement = (citations, citeElement) => {
    const classes = citeElement.classList;
    for (let k = 0; k < classes.length; k++) {
        if (classes[k] !== 'citation') {
            citations.add(classes[k]);
        }
    }
};

const addCitationsToContent = (citations, searchElement) => {
    if (searchElement.tagName === 'CITE') {
        collectCitationsOfOneCiteElement(citations, searchElement);
    } else {
        const citeElements = searchElement.querySelectorAll('cite');
        for (let i = 0; i < citeElements.length; i++) {
{'fraction_non_alphanumeric': 0.14020054694621695, 'fraction_numerical': 0.0021877848678213308, 'mean_word_length': 3.409967845659164, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6345177', 'n_tokens_mistral': 1646, 'n_tokens_neox': 1621, 'n_words': 359}