<commit_before>const moment = require('moment');
const sha256 = require('sha256');
const rp = require('request-promise');
const Nightmare = require('nightmare');
const nightmare = Nightmare({ show: false, typeInterval: 2, waitTimeout: 5000 });

const URL = 'http://www.madr.gov.ro/transparenta-decizionala/proiecte-de-acte-normative.html';
const YEAR_THRESHOLD = 2017;

const API_TOKEN = process.env['API_TOKEN'];

function parsePage(firstFlag) {
    if (firstFlag) {
        nightmare
          .goto(URL);
    } else {
        nightmare
          .click('.pagination .next');
    }

    nightmare
      .wait("#itemListPrimary .itemContainer")
      .evaluate(()=> {
        let itemsList = [], items = [... document.querySelectorAll('#itemListPrimary .itemContainer')];
        for (let item of items) {
            let introText = item.querySelector(".catItemIntroText");
            let emHTML = introText.querySelector("em");
            if (emHTML !== null) {
                emHTML.remove();
            }

            let returnObj = {
                title: introText.innerText,
                date: item.querySelector(".catItemDateCreated").innerText,
                documents: [...item.querySelectorAll(".catItemAttachments a")].map((elem)=> {
                    return {
                        type: "act",//elem.title,
                        url: elem.href
                    }

                })
            };

            itemsList.push(returnObj);
        }
        return itemsList;
      })
      .then((result) => {

        let itemsList = [];

        for(let val of result) {
            let date = moment(val.date, 'ddd, DD MMMM YYYY HH:mm', 'ro');
            let year = date.year();
            if (year < YEAR_THRESHOLD) {
                console.log("halt!");
                nightmare.halt();
                return;
            }

            let identifier = sha256(val.title);
            val.date = date.format('YYYY-MM-DD');
            val.identifier = identifier;<commit_msg>issuer, description<commit_after>const moment = require('moment');
const sha256 = require('sha256');
const rp = require('request-promise');
const Nightmare = require('nightmare');
const nightmare = Nightmare({ show: false, typeInterval: 2, waitTimeout: 5000 });

const URL = 'http://www.madr.gov.ro/transparenta-decizionala/proiecte-de-acte-normative.html';
const YEAR_THRESHOLD = 2017;

const API_TOKEN = process.env['API_TOKEN'];

function parsePage(firstFlag) {
    if (firstFlag) {
        nightmare
          .goto(URL);
    } else {
        nightmare
          .click('.pagination .next');
    }

    nightmare
      .wait("#itemListPrimary .itemContainer")
      .evaluate(()=> {
        let itemsList = [], items = [... document.querySelectorAll('#itemListPrimary .itemContainer')];
        for (let item of items) {
            let introText = item.querySelector(".catItemIntroText");
            let emHTML = introText.querySelector("em");
            if (emHTML !== null) {
                emHTML.remove();
            }

            let returnObj = {
                title: introText.innerText,
                date: item.querySelector(".catItemDateCreated").innerText,
                documents: [...item.querySelectorAll(".catItemAttachments a")].map((elem)=> {
                    return {
                        type: "act",//elem.title,
                        url: elem.href
                    }

                })
            };

            itemsList.push(returnObj);
        }
        return itemsList;
      })
      .then((result) => {

        let itemsList = [];

        for(let val of result) {
            let date = moment(val.date, 'ddd, DD MMMM YYYY HH:mm', 'ro');
            let year = date.year();
            if (year < YEAR_THRESHOLD) {
                console.log("halt!");
                nightmare.halt();
                return;
            }

            let identifier = sha256(val.title);
            val.date = date.format('YYYY-MM-DD');
            val.identifier = identifier;
            val.issuer = 'agricultura';
            val.description = val.title;
{'fraction_non_alphanumeric': 0.1338181818181818, 'fraction_numerical': 0.008727272727272728, 'mean_word_length': 1.7488341105929381, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3995919', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1125, 'n_words': 230}