<commit_before>const fs = require('fs');
const URL = require('url');
const { JSDOM } = require('jsdom');
const filenamify = require('filenamify-url');
const baseFetch = require('node-fetch');
const Response = require('node-fetch').Response;
const rimraf = require('rimraf');
const respecWriter = require("respec/tools/respecDocWriter").fetchAndWrite;
var config = null;
try {
    config = require('./config.json');
}
catch (e) {
    config = {};
};


// The list of URLs that are being fetched and that should soon
// be available from the cache, together with the Promise to have
// fetched them
const pendingFetches = {};

// The list of URLs that have been fetched (and that exist in the cache)
// during this crawl. Used as a basic "max-age" mechanism to avoid sending
// multiple requests to the same URL per crawl
const fetchedUrls = [];

// Reset the cache folder only once
var cacheFolderReset = false;


/**
 * Wrapper around the baseFetch function that returns the response from the
 * local cache if one is found.
 *
 * TODO: use encoding specified in content-type header for file operations?
 *
 * @function
 * @private
 * @param {String} url The URL to retrieve
 * @param {Object} options Fetch options, include specific HTTP headers to
 *   send along with the request.
 * @return {Promise<Response>} The promise to get an HTTP response
 */
function fetch(url, options) {
    const cacheFilename = 'cache/' + filenamify(url);
    const cacheHeadersFilename = cacheFilename + '.headers';
    options = options || {};

    if (!cacheFolderReset) {
        cacheFolderReset = true;
        if (config.resetCache) {
            // NB: using "sync" versions to avoid having to deal with
            // parallel requests that could start using the contents of
            // the cache before it has been fully reset.
            rimraf.sync('cache/*');
        }
    }

    function checkCacheFolder() {
        return new Promise((resolve, reject) => {
            fs.stat('cache', (err, stat) => {
                if (err && (err.code === 'ENOENT')) {
                    fs.mkdir('cache', err => {
                        if (err && err.code === 'EEXIST') {
                            // Someone created the folder in the meantime
                            resolve();
                        }
                        else if (err) {
                            reject(err);
                        }
                        else {
                            resolve();
                        }
                    });
                }
                else if (err) {
                    reject(err);
                }
                else if (stat.isDirectory()) {
                    resolve();
                }
                else {
                    reject(new Error('Looking for a "cache" folder but found a "cache" file instead'));
                }
            });
        });
    }

    function checkPendingFetch() {
        if (pendingFetches[url]) {
            return pendingFetches[url];
        }
        else {
            return Promise.resolve();
        }
    }

    function readHeadersFromCache() {
        return new Promise((resolve, reject) => {
            fs.readFile(cacheHeadersFilename, 'utf8', function (err, data) {
                var headers = null;
                if (!err) {
                    try {
                        headers = JSON.parse(data);
                    }
                    catch (e) {
                    }
                }
                resolve(headers);
            });
        });
    }

    function readFromCache() {
        return new Promise((resolve, reject) => {
            fs.readFile(cacheHeadersFilename, 'utf8', function (err, data) {
                if (err) {
                    return reject(err);
                }
                var headers = null;
                try {
                    headers = JSON.parse(data);
                }
                catch (e) {
                    return reject(e);
                }
                resolve(headers);
            });
        }).then(headers => new Promise((resolve, reject) => {
            fs.readFile(cacheFilename, 'utf8', function (err, data) {
                if (err) {
                    return reject(err);
                }
                resolve(new Response(data, {
                    url,
                    status: 200,
                    headers
                }));
            });
        }));
    }

    function saveToCacheIfNeeded(response) {
        function saveHeaders() {
            return new Promise((resolve, reject) => {
                const headers = {};
                response.headers.forEach((value, header) => headers[header] = value);
                fs.writeFile(
                    cacheHeadersFilename,
                    JSON.stringify(headers, null, 2),
                    'utf8',
                    err => (err ? reject(err) : resolve()));
            });
        }

        function saveBody() {
            return response.text()
                .then(data => new Promise((resolve, reject) => {
                    fs.writeFile(cacheFilename, data, 'utf8',
                        err => (err ? reject(err) : resolve()));
                }));
        }

        if (response.status === 304) {
            // Response is the one we have in cache
            return;
        }
        else {
            return saveHeaders().then(saveBody);
        }
    }

    function conditionalFetch(prevHeaders) {
        if ((prevHeaders && config.avoidNetworkRequests) || fetchedUrls[url]) {
            console.log('Fetch (from cache): ' + url);
            return readFromCache();
        }

        options.headers = options.headers || {};
        if (prevHeaders && prevHeaders['last-modified']) {
            options.headers['If-Modified-Since'] = prevHeaders['last-modified'];
        }
        if (prevHeaders && prevHeaders.etag) {
            options.headers['If-None-Match'] = prevHeaders.etag;
        }

        if (options.headers['If-Modified-Since'] ||
            options.headers['If-None-Match']) {
            console.log('Fetch (conditional request): ' + url);
        }
        else {
            console.log('Fetch: ' + url);
        }
        return baseFetch(url, options)
            .then(saveToCacheIfNeeded)
            .then(readFromCache);
    }

    return checkCacheFolder()
        .then(checkPendingFetch)
        .then(() => {
            pendingFetches[url] = readHeadersFromCache()
                .then(conditionalFetch)
                .then(response => {
                    delete pendingFetches[url];
                    fetchedUrls[url] = true;
                    return response;
                });

            return pendingFetches[url];
        });
}


/**
 * Load the given specification.
 *
 * @function
 * @public
 * @param {String} url The URL of the specification to load
 * @return {Promise} The promise to get a window object once the spec has
 *   been loaded with jsdom.
 */
function loadSpecification(url) {
    return fetch(url).then(response => new Promise((resolve, reject) => {
        response.text().then(html => {
            const {window} = new JSDOM(html, {url: response.url});
            const doc = window.document;
            // ReSpec doc
            if (doc.head.querySelector("script[src*='respec']")) {
                // this does another network fetch :(
                return respecWriter(url, '', {}, 20000).then(function(html) {
                    const dom = new JSDOM(html, {url: response.url});
                    resolve(dom.window);
                }).catch(reject);<commit_msg>Be more lenient in looking for respec script<commit_after>const fs = require('fs');
const URL = require('url');
const { JSDOM } = require('jsdom');
const filenamify = require('filenamify-url');
const baseFetch = require('node-fetch');
const Response = require('node-fetch').Response;
const rimraf = require('rimraf');
const respecWriter = require("respec/tools/respecDocWriter").fetchAndWrite;
var config = null;
try {
    config = require('./config.json');
}
catch (e) {
    config = {};
};


// The list of URLs that are being fetched and that should soon
// be available from the cache, together with the Promise to have
// fetched them
const pendingFetches = {};

// The list of URLs that have been fetched (and that exist in the cache)
// during this crawl. Used as a basic "max-age" mechanism to avoid sending
// multiple requests to the same URL per crawl
const fetchedUrls = [];

// Reset the cache folder only once
var cacheFolderReset = false;


/**
 * Wrapper around the baseFetch function that returns the response from the
 * local cache if one is found.
 *
 * TODO: use encoding specified in content-type header for file operations?
 *
 * @function
 * @private
 * @param {String} url The URL to retrieve
 * @param {Object} options Fetch options, include specific HTTP headers to
 *   send along with the request.
 * @return {Promise<Response>} The promise to get an HTTP response
 */
function fetch(url, options) {
    const cacheFilename = 'cache/' + filenamify(url);
    const cacheHeadersFilename = cacheFilename + '.headers';
    options = options || {};

    if (!cacheFolderReset) {
        cacheFolderReset = true;
        if (config.resetCache) {
            // NB: using "sync" versions to avoid having to deal with
            // parallel requests that could start using the contents of
            // the cache before it has been fully reset.
            rimraf.sync('cache/*');
        }
    }

    function checkCacheFolder() {
        return new Promise((resolve, reject) => {
            fs.stat('cache', (err, stat) => {
                if (err && (err.code === 'ENOENT')) {
                    fs.mkdir('cache', err => {
                        if (err && err.code === 'EEXIST') {
                            // Someone created the folder in the meantime
                            resolve();
                        }
                        else if (err) {
                            reject(err);
                        }
                        else {
                            resolve();
                        }
                    });
                }
                else if (err) {
                    reject(err);
                }
                else if (stat.isDirectory()) {
                    resolve();
                }
                else {
                    reject(new Error('Looking for a "cache" folder but found a "cache" file instead'));
                }
            });
        });
    }

    function checkPendingFetch() {
        if (pendingFetches[url]) {
            return pendingFetches[url];
        }
        else {
            return Promise.resolve();
        }
    }

    function readHeadersFromCache() {
        return new Promise((resolve, reject) => {
            fs.readFile(cacheHeadersFilename, 'utf8', function (err, data) {
                var headers = null;
                if (!err) {
                    try {
                        headers = JSON.parse(data);
                    }
                    catch (e) {
                    }
                }
                resolve(headers);
            });
        });
    }

    function readFromCache() {
        return new Promise((resolve, reject) => {
            fs.readFile(cacheHeadersFilename, 'utf8', function (err, data) {
                if (err) {
                    return reject(err);
                }
                var headers = null;
                try {
                    headers = JSON.parse(data);
                }
                catch (e) {
                    return reject(e);
                }
                resolve(headers);
            });
        }).then(headers => new Promise((resolve, reject) => {
            fs.readFile(cacheFilename, 'utf8', function (err, data) {
                if (err) {
                    return reject(err);
                }
                resolve(new Response(data, {
                    url,
                    status: 200,
                    headers
                }));
            });
        }));
    }

    function saveToCacheIfNeeded(response) {
        function saveHeaders() {
            return new Promise((resolve, reject) => {
                const headers = {};
                response.headers.forEach((value, header) => headers[header] = value);
                fs.writeFile(
                    cacheHeadersFilename,
                    JSON.stringify(headers, null, 2),
                    'utf8',
                    err => (err ? reject(err) : resolve()));
            });
        }

        function saveBody() {
            return response.text()
                .then(data => new Promise((resolve, reject) => {
                    fs.writeFile(cacheFilename, data, 'utf8',
                        err => (err ? reject(err) : resolve()));
                }));
        }

        if (response.status === 304) {
            // Response is the one we have in cache
            return;
        }
        else {
            return saveHeaders().then(saveBody);
        }
    }

    function conditionalFetch(prevHeaders) {
        if ((prevHeaders && config.avoidNetworkRequests) || fetchedUrls[url]) {
            console.log('Fetch (from cache): ' + url);
            return readFromCache();
        }

        options.headers = options.headers || {};
        if (prevHeaders && prevHeaders['last-modified']) {
            options.headers['If-Modified-Since'] = prevHeaders['last-modified'];
        }
        if (prevHeaders && prevHeaders.etag) {
            options.headers['If-None-Match'] = prevHeaders.etag;
        }

        if (options.headers['If-Modified-Since'] ||
            options.headers['If-None-Match']) {
            console.log('Fetch (conditional request): ' + url);
        }
        else {
            console.log('Fetch: ' + url);
        }
        return baseFetch(url, options)
            .then(saveToCacheIfNeeded)
            .then(readFromCache);
    }

    return checkCacheFolder()
        .then(checkPendingFetch)
        .then(() => {
            pendingFetches[url] = readHeadersFromCache()
                .then(conditionalFetch)
                .then(response => {
                    delete pendingFetches[url];
                    fetchedUrls[url] = true;
                    return response;
                });

            return pendingFetches[url];
        });
}


/**
 * Load the given specification.
 *
 * @function
 * @public
 * @param {String} url The URL of the specification to load
 * @return {Promise} The promise to get a window object once the spec has
 *   been loaded with jsdom.
 */
function loadSpecification(url) {
    return fetch(url).then(response => new Promise((resolve, reject) => {
        response.text().then(html => {
            const {window} = new JSDOM(html, {url: response.url});
            const doc = window.document;
            // ReSpec doc
            if (doc.querySelector("script[src*='respec']")) {
                // this does another network fetch :(
                return respecWriter(url, '', {}, 20000).then(function(html) {
                    const dom = new JSDOM(html, {url: response.url});
                    resolve(dom.window);
                }).catch(reject);
{'fraction_non_alphanumeric': 0.11518426878499805, 'fraction_numerical': 0.002213829925771585, 'mean_word_length': 1.4768585711981939, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '26977', 'n_tokens_mistral': 3918, 'n_tokens_neox': 3711, 'n_words': 1048}