<commit_before>var refParser = require('./parse-references');
var webidlExtractor = require('./extract-webidl');
var loadSpecification = require('./util').loadSpecification;
var webidlParser = require('./parse-webidl');
var fetch = require('./util').fetch;
var fs = require('fs');

/**
 * Extracts the title of the loaded document
 */
function titleExtractor(window) {
    var title = window.document.querySelector("title");
    if (window.location.href === 'https://html.spec.whatwg.org/multipage/workers.html') {
        // Web Worker ED is a page of the HTML Living Standard.
        // Report the appropriate title (crawler will still be confused because
        // it won't find any normative references at the end of this page)
        return 'Web Workers';
    }
    else if (title) {
        return title.textContent.trim();
    }
    else {
        return '[No title found for ' + window.location.href + ']';
    }
}


/**
 * Return the given URL along with the W3C shortname for that specification
 *
 * @function
 * @private
 * @param {String} url The URL to enrich
 * @return {{url: String, shortname: String} The beginning of a spec description
 *   structure that contains the URL along with the shortname, when possible
 */
function getShortname(url) {
    var res = { url };
    if (!url.match(/www.w3.org\/TR\//)) {
        return res;
    }
    if (url.match(/TR\/[0-9]+\//)) {
        // dated version
        var statusShortname = url.split('/')[5];
        res.shortname = statusShortname.split('-').slice(1, -1).join('-');
        return res;
    }
    // latest version
    res.shortname = url.split('/')[4];
    return res;
}


/**
 * Enrich the spec description with the URL of the latest editor's draft,
 * and with the title of the specification, provided that the specification
 * is a W3C spec.
 *
 * @function
 * @param {Object} spec Spec description structure (only the URL is useful)
 * @return {Objec} The same structure, enriched with the URL of the latest
 *   draft when one is found
 */
function getSpecFromW3CApi(spec) {
    var shortname = spec.shortname;
    var config = require('./config.json');
    var options = {
        headers: {
            Authorization: 'W3C-API apikey="' + config.w3cApiKey + '"'
        }
    };

    // Note the mapping between some of the specs (e.g. HTML5.1 and HTML5)
    // is hardcoded below. In an ideal world, it would be easy to get that
    // info from the W3C API.
    spec.versions = [];
    function addKnownVersions() {
        if (!spec.versions.includes(spec.url)) {
            spec.versions.push(spec.url);
        }
        if (spec.latest && !spec.versions.includes(spec.latest)) {
            spec.versions.push(spec.latest);
        }
        if (shortname === 'html51') {
            spec.versions.push('https://www.w3.org/TR/html5/');
            spec.versions.push('https://html.spec.whatwg.org/');
            spec.versions.push('https://html.spec.whatwg.org/multipage/');
        }
        if (spec.url === 'https://dom.spec.whatwg.org/') {
            spec.versions.push('https://www.w3.org/TR/dom/');
        }
    }

    if (!shortname) {
        addKnownVersions();
        return spec;
    }
    var bogusEditorDraft = ['webmessaging', 'eventsource', 'webstorage', 'progress-events', 'payment-method-basic-card', 'payment-request'];
    var unparseableEditorDraft = ['requestidlecallback', 'beacon'];
    if ((bogusEditorDraft.indexOf(shortname) !== -1)
        || (unparseableEditorDraft.indexOf(shortname) !== -1)) {
        spec.latest = 'https://www.w3.org/TR/' + shortname;
        addKnownVersions();
        return spec;
    }
    return fetch('https://api.w3.org/specifications/' + shortname, options)
        .then(r =>  r.json())
        .then(s => fetch(s._links['latest-version'].href, options))
        .then(r => r.json())
        .then(s => {
            spec.latest = (s['editor-draft'] ? s['editor-draft'] : s.uri);
            if (!spec.versions.includes(s.uri)) {
                spec.versions.push(s.uri);
            }
            return spec;
        })<commit_msg>Added UI Events spec to list of bogus ED<commit_after>var refParser = require('./parse-references');
var webidlExtractor = require('./extract-webidl');
var loadSpecification = require('./util').loadSpecification;
var webidlParser = require('./parse-webidl');
var fetch = require('./util').fetch;
var fs = require('fs');

/**
 * Extracts the title of the loaded document
 */
function titleExtractor(window) {
    var title = window.document.querySelector("title");
    if (window.location.href === 'https://html.spec.whatwg.org/multipage/workers.html') {
        // Web Worker ED is a page of the HTML Living Standard.
        // Report the appropriate title (crawler will still be confused because
        // it won't find any normative references at the end of this page)
        return 'Web Workers';
    }
    else if (title) {
        return title.textContent.trim();
    }
    else {
        return '[No title found for ' + window.location.href + ']';
    }
}


/**
 * Return the given URL along with the W3C shortname for that specification
 *
 * @function
 * @private
 * @param {String} url The URL to enrich
 * @return {{url: String, shortname: String} The beginning of a spec description
 *   structure that contains the URL along with the shortname, when possible
 */
function getShortname(url) {
    var res = { url };
    if (!url.match(/www.w3.org\/TR\//)) {
        return res;
    }
    if (url.match(/TR\/[0-9]+\//)) {
        // dated version
        var statusShortname = url.split('/')[5];
        res.shortname = statusShortname.split('-').slice(1, -1).join('-');
        return res;
    }
    // latest version
    res.shortname = url.split('/')[4];
    return res;
}


/**
 * Enrich the spec description with the URL of the latest editor's draft,
 * and with the title of the specification, provided that the specification
 * is a W3C spec.
 *
 * @function
 * @param {Object} spec Spec description structure (only the URL is useful)
 * @return {Objec} The same structure, enriched with the URL of the latest
 *   draft when one is found
 */
function getSpecFromW3CApi(spec) {
    var shortname = spec.shortname;
    var config = require('./config.json');
    var options = {
        headers: {
            Authorization: 'W3C-API apikey="' + config.w3cApiKey + '"'
        }
    };

    // Note the mapping between some of the specs (e.g. HTML5.1 and HTML5)
    // is hardcoded below. In an ideal world, it would be easy to get that
    // info from the W3C API.
    spec.versions = [];
    function addKnownVersions() {
        if (!spec.versions.includes(spec.url)) {
            spec.versions.push(spec.url);
        }
        if (spec.latest && !spec.versions.includes(spec.latest)) {
            spec.versions.push(spec.latest);
        }
        if (shortname === 'html51') {
            spec.versions.push('https://www.w3.org/TR/html5/');
            spec.versions.push('https://html.spec.whatwg.org/');
            spec.versions.push('https://html.spec.whatwg.org/multipage/');
        }
        if (spec.url === 'https://dom.spec.whatwg.org/') {
            spec.versions.push('https://www.w3.org/TR/dom/');
        }
    }

    if (!shortname) {
        addKnownVersions();
        return spec;
    }
    var bogusEditorDraft = ['webmessaging', 'eventsource', 'webstorage', 'progress-events', 'payment-method-basic-card', 'payment-request', 'uievents'];
    var unparseableEditorDraft = ['requestidlecallback', 'beacon'];
    if ((bogusEditorDraft.indexOf(shortname) !== -1)
        || (unparseableEditorDraft.indexOf(shortname) !== -1)) {
        spec.latest = 'https://www.w3.org/TR/' + shortname;
        addKnownVersions();
        return spec;
    }
    return fetch('https://api.w3.org/specifications/' + shortname, options)
        .then(r =>  r.json())
        .then(s => fetch(s._links['latest-version'].href, options))
        .then(r => r.json())
        .then(s => {
            spec.latest = (s['editor-draft'] ? s['editor-draft'] : s.uri);
            if (!spec.versions.includes(s.uri)) {
                spec.versions.push(s.uri);
            }
            return spec;
        })
{'fraction_non_alphanumeric': 0.15029816234635512, 'fraction_numerical': 0.006084945843981989, 'mean_word_length': 2.4471476510067114, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 16, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '601204', 'n_tokens_mistral': 2513, 'n_tokens_neox': 2466, 'n_words': 653}