<commit_before>/* eslint no-console: 0 */
'use strict';

const phantom = require('phantom');

const logger = require('../logger.js');


function* blockResourceLoading(page) {
    yield page.property('onResourceRequested', function(requestData, request) {
        var BLOCKED_RESOURCES = [
            /\.gif/gi,
            /\.png/gi,
            /\.css/gi
        ];
        var is_blacklisted_resource = BLOCKED_RESOURCES.some(function(r) {
            return r.test(requestData['url']);
        });

        if (is_blacklisted_resource) {
            request.abort();
        }
    });
}

function* fetchPageContent(url, instance, block_assets=true) {
    let is_local_instance = false;
    if (!instance) {
        instance = yield * initPhantomInstance();
        is_local_instance = true;
    }

    const page = yield instance.createPage();
    if (block_assets) {
        yield * blockResourceLoading(page);
    }

    yield page.open(url);
    let html = yield page.property('content');
    yield page.close();
    if (is_local_instance) {
        instance.exit();
    }
    return html;
}

function* initPhantomInstance() {
    const instance = yield phantom.create();
    setupPhantomCleanupOnNodeProcessTermination(instance);

    return instance;
}

function setupPhantomCleanupOnNodeProcessTermination(instance) {
    const phantomInstanceKill = phantomKill.bind(null, instance);

    // On app closing
    process.on('exit', phantomInstanceKill);

    // Catch CTRL+C
    process.on('SIGINT', phantomInstanceKill);

    process.on('SIGTERM', phantomInstanceKill);
    process.on('SIGHUP', phantomInstanceKill);

    // On uncaught exceptions
    process.on('uncaughtException', phantomInstanceKill);

    // On uncaught rejected promises
    process.on('unhandledRejection', phantomInstanceKill);<commit_msg>Call process.exit on SIGINT, SIGTERM and SIGHUP instead of phantomInstanceKill.<commit_after>/* eslint no-console: 0 */
'use strict';

const phantom = require('phantom');

const logger = require('../logger.js');


function* blockResourceLoading(page) {
    yield page.property('onResourceRequested', function(requestData, request) {
        var BLOCKED_RESOURCES = [
            /\.gif/gi,
            /\.png/gi,
            /\.css/gi
        ];
        var is_blacklisted_resource = BLOCKED_RESOURCES.some(function(r) {
            return r.test(requestData['url']);
        });

        if (is_blacklisted_resource) {
            request.abort();
        }
    });
}

function* fetchPageContent(url, instance, block_assets=true) {
    let is_local_instance = false;
    if (!instance) {
        instance = yield * initPhantomInstance();
        is_local_instance = true;
    }

    const page = yield instance.createPage();
    if (block_assets) {
        yield * blockResourceLoading(page);
    }

    yield page.open(url);
    let html = yield page.property('content');
    yield page.close();
    if (is_local_instance) {
        instance.exit();
    }
    return html;
}

function* initPhantomInstance() {
    const instance = yield phantom.create();
    setupPhantomCleanupOnNodeProcessTermination(instance);

    return instance;
}

function setupPhantomCleanupOnNodeProcessTermination(instance) {
    const phantomInstanceKill = phantomKill.bind(null, instance);

    // On app closing
    process.on('exit', phantomInstanceKill);

    // Catch CTRL+C
    process.on('SIGINT', process.exit);

    process.on('SIGTERM', process.exit);
    process.on('SIGHUP', process.exit);

    // On uncaught exceptions
    process.on('uncaughtException', phantomInstanceKill);

    // On uncaught rejected promises
    process.on('unhandledRejection', phantomInstanceKill);
{'fraction_non_alphanumeric': 0.12137632078027635, 'fraction_numerical': 0.000541858574911948, 'mean_word_length': 2.9913513513513514, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3673882', 'n_tokens_mistral': 1148, 'n_tokens_neox': 1043, 'n_words': 218}