<commit_before>/**
 * @fileoverview This hint verifies that all links and resources the page
 * uses are available online. Checks for 404, 410, 500 or 503 status
 */

import { URL } from 'url';
import { HintContext } from 'hint/dist/src/lib/hint-context';
import {
    IHint,
    ElementFound,
    IAsyncHTMLElement
} from 'hint/dist/src/lib/types';
import { debug as d } from 'hint/dist/src/lib/utils/debug';
import isRegularProtocol from 'hint/dist/src/lib/utils/network/is-regular-protocol';
import { Requester } from '@hint/utils-connector-tools/dist/src/requester';
import { IAsyncHTMLDocument, NetworkData, TraverseEnd } from 'hint/dist/src/lib/types';
import { CoreOptions } from 'request';

import meta from './meta';

const debug: debug.IDebugger = d(__filename);

/*
 * ------------------------------------------------------------------------------
 * Public
 * ------------------------------------------------------------------------------
 */

export default class NoBrokenLinksHint implements IHint {
    public static readonly meta = meta;

    public constructor(context: HintContext) {

        const options: CoreOptions = { method: context.hintOptions && context.hintOptions.method ? context.hintOptions.method : 'GET' };
        const requester = new Requester(options);
        const brokenStatusCodes = [404, 410, 500, 503];

        /** Stores the elements with their URLs which have been collected while traversing the page. */
        const collectedElementsWithURLs: [IAsyncHTMLElement, string[]][] = [];

        /** Stores the URLs and it's response status codes */
        const fetchedURLs: any[] = [];

        /** Returns an array with all the URLs in the given `srcset` attribute or an empty string if none. */
        const parseSrcSet = (srcset: string | null): string[] => {
            if (!srcset) {
                return [];
            }

            const urls = srcset
                .split(',')
                .map((entry) => {
                    return entry.trim().split(' ')[0].trim();
                });

            return urls;
        };

        /**
         * The callback to handle rejection returned from the `head` method
         * When DNS resolution fails, it will be handled here (ex : https://thissitedoesnotexist.com/ )
         */
        const handleRejection = (error: any, url: string, element: IAsyncHTMLElement) => {
            debug(`Error accessing {$absoluteURL}. ${JSON.stringify(error)}`);

            if (typeof error === 'string' && error.toLowerCase().includes('loop')) {
                return context.report(url, error, { element });
            }

            return context.report(url, 'Broken link found (domain not found).', { element });
        };

        const isDNSOnlyResourceHint = (element: IAsyncHTMLElement): boolean => {
            if (element.nodeName !== 'LINK') {<commit_msg>Fix: Debug message<commit_after>/**
 * @fileoverview This hint verifies that all links and resources the page
 * uses are available online. Checks for 404, 410, 500 or 503 status
 */

import { URL } from 'url';
import { HintContext } from 'hint/dist/src/lib/hint-context';
import {
    IHint,
    ElementFound,
    IAsyncHTMLElement
} from 'hint/dist/src/lib/types';
import { debug as d } from 'hint/dist/src/lib/utils/debug';
import isRegularProtocol from 'hint/dist/src/lib/utils/network/is-regular-protocol';
import { Requester } from '@hint/utils-connector-tools/dist/src/requester';
import { IAsyncHTMLDocument, NetworkData, TraverseEnd } from 'hint/dist/src/lib/types';
import { CoreOptions } from 'request';

import meta from './meta';

const debug: debug.IDebugger = d(__filename);

/*
 * ------------------------------------------------------------------------------
 * Public
 * ------------------------------------------------------------------------------
 */

export default class NoBrokenLinksHint implements IHint {
    public static readonly meta = meta;

    public constructor(context: HintContext) {

        const options: CoreOptions = { method: context.hintOptions && context.hintOptions.method ? context.hintOptions.method : 'GET' };
        const requester = new Requester(options);
        const brokenStatusCodes = [404, 410, 500, 503];

        /** Stores the elements with their URLs which have been collected while traversing the page. */
        const collectedElementsWithURLs: [IAsyncHTMLElement, string[]][] = [];

        /** Stores the URLs and it's response status codes */
        const fetchedURLs: any[] = [];

        /** Returns an array with all the URLs in the given `srcset` attribute or an empty string if none. */
        const parseSrcSet = (srcset: string | null): string[] => {
            if (!srcset) {
                return [];
            }

            const urls = srcset
                .split(',')
                .map((entry) => {
                    return entry.trim().split(' ')[0].trim();
                });

            return urls;
        };

        /**
         * The callback to handle rejection returned from the `head` method
         * When DNS resolution fails, it will be handled here (ex : https://thissitedoesnotexist.com/ )
         */
        const handleRejection = (error: any, url: string, element: IAsyncHTMLElement) => {
            debug(`Error accessing ${url}. ${JSON.stringify(error)}`);

            if (typeof error === 'string' && error.toLowerCase().includes('loop')) {
                return context.report(url, error, { element });
            }

            return context.report(url, 'Broken link found (domain not found).', { element });
        };

        const isDNSOnlyResourceHint = (element: IAsyncHTMLElement): boolean => {
            if (element.nodeName !== 'LINK') {
{'fraction_non_alphanumeric': 0.17162232655190401, 'fraction_numerical': 0.00869414014953921, 'mean_word_length': 2.1008086253369274, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7237125', 'n_tokens_mistral': 1543, 'n_tokens_neox': 1462, 'n_words': 458}