<commit_before>var nightmareConfig = { show: false },
    cheerio = require('cheerio'),
    parseFunction = require('./parseProject');

var URL = 'http://economie.gov.ro/transparenta-decizionala/proiecte-in-dezbatere-publica',
    BASE = 'http://economie.gov.ro';

function parseItem(item) {
    parseFunction(item);
}

require('nightmare')(nightmareConfig)
    .goto(URL)
    .wait('body')
    .evaluate(function () {
        return document.querySelector('.pagination').innerHTML;
    })
    .end()
    .then(function (result) {
        var pages = [URL];
        cheerio.load(result)('a').each(function (i, link) {
            pages.push(BASE + link.attribs['href']);
        });

        return pages;
    })
    .then(function (result) {
        result.forEach(function (page) {
            require('nightmare')(nightmareConfig)
                .goto(page)
                .wait('body')
                .evaluate(function () {
                    return document.querySelector('.blog').innerHTML;
                })
                .end()<commit_msg>passing base url<commit_after>var nightmareConfig = { show: false },
    cheerio = require('cheerio'),
    parseFunction = require('./parseProject');

var URL = 'http://economie.gov.ro/transparenta-decizionala/proiecte-in-dezbatere-publica',
    BASE = 'http://economie.gov.ro';

function parseItem(item) {
    parseFunction(item, BASE);
}

require('nightmare')(nightmareConfig)
    .goto(URL)
    .wait('body')
    .evaluate(function () {
        return document.querySelector('.pagination').innerHTML;
    })
    .end()
    .then(function (result) {
        var pages = [URL];
        cheerio.load(result)('a').each(function (i, link) {
            pages.push(BASE + link.attribs['href']);
        });

        return pages;
    })
    .then(function (result) {
        result.forEach(function (page) {
            require('nightmare')(nightmareConfig)
                .goto(page)
                .wait('body')
                .evaluate(function () {
                    return document.querySelector('.blog').innerHTML;
                })
                .end()
{'fraction_non_alphanumeric': 0.1562057574327513, 'fraction_numerical': 0.0, 'mean_word_length': 2.202416918429003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '755142', 'n_tokens_mistral': 633, 'n_tokens_neox': 620, 'n_words': 94}