<commit_before>/*
    Copyright 2010 <NAME>

    This file is part of KaVeikiaValdzia.lt website.

    PolicyFeed is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    PolicyFeed is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with PolicyFeed.  If not, see <http://www.gnu.org/licenses/>.
*/


// Requirements:
var Failures = require("PolicyFeed/Crawler/Failures");
var gluestick = require("gluestick");
var htmlunit = require("htmlunit");
var Queue = require("PolicyFeed/Crawler/Queue");
var organizations = require("KaVeikiaValdzia/tags/organizations");
var parser_utils = require("./utils");
var ringo_dates = require("ringo/utils/dates");

// Extends:
gluestick.extendModule(exports, "PolicyFeed/Crawler/Parser");
gluestick.extendModule(exports, "KaVeikiaValdzia/parsers/lrs.lt-common");

exports.disabled = false;

// Config:
exports.feed_url = [
    "http://www.lrs.lt/pls/proj/dokpaieska.rezult_l"
    ,"http://www.lrs.lt/pls/proj/dokpaieska.rezult_l?p_tr1=2&p_tr2=2&p_fix=y&p_gov=n&p_no=2"
    ,"http://www.lrs.lt/pls/proj/dokpaieska.rezult_l?p_tr1=2&p_tr2=2&p_fix=y&p_gov=n&p_no=3"
    ];


/**
 *
 */
exports.extractFeedItems = function(page) {
    this.validateFeedPage(page);
    htmlunit.fixPageUrls(page);

    var items = page.getByXPath('/html/body/div/table/tbody/tr[2]/td/table/tbody/tr/td/align/table[2]/tbody/tr').toArray();

    items.shift();
    if (items.length < 1) {
        return [];
    } else {
        return items.map(this.parseFeedItem()).filter(function(item) { return item });
    }
}


/**
 *
 */
exports.parseFeedItem = function(item) {
    var name = this.name;
    var doc_template = this.doc_template;
    var error = this.error;
    var orgmap = organizations.getNameMap();

    return function(item) {

        var result = {};
        var teises_aktas = {};

        var num = item.getFirstByXPath('./td[1]').asText();
        var content = item.getFirstByXPath('./td[2]');
        var published = item.getFirstByXPath('./td[3]').asText();
        var numeris = item.getFirstByXPath('./td[4]').asText();
        if (numeris)
            teises_aktas.numeris = numeris;
        var type = item.getFirstByXPath('./td[5]').asText();

        // url and title:
        var tspan = content.getFirstByXPath('./span');
        if (!tspan&& (num % 30 == 0)) {
            // Link is always missing for item #30 (might be a HtmlUnit bug):
            return false;
        }
        var tlink = tspan.getFirstByXPath('./a');
        var title = tlink.asText();
        var url = parser_utils.getCanonicalLrsUrl( tlink.getHrefAttribute().toString() );
        tspan.remove();

        var small_links = content.getByXPath('./small/a');
        if (small_links) {
            small_links = small_links.toArray();
            for each (var slink in small_links) {
                switch (slink.asText()) {
                    case "Priedai":
                        teises_aktas.priedai = slink.getHrefAttribute().toString();
                    break;
                    case "[Susiję dokumentai]":
                        teises_aktas.susije = slink.getHrefAttribute().toString();
                    break;
                    default:
                        Failures.write(url, { parser: name, error: "Unrecognized small link",
                                text: slink.asText(),
                                link: slink.getHrefAttribute().toString()
                                });
                }
                slink.remove();
            }
        }

        // get non-empty content lines:
        content = content.asText().split(/[\n;]/).filter(function(part) { return part.trim(); });

        var author = "";
        for each (var line in content) {
            line = line.split(/[:,\/]/).map(function(part) { return part.trim() });

            switch (line[0]) {
                case "Pateikė":
                case "Priėmė":
                case "Parengė":
                    if (line[1].slice(0, 12) == "Seimo narys " || line[1].slice(0, 11) == "Seimo narė ") {<commit_msg>Fixed data structure for "Unrecognized small link" failures.<commit_after>/*
    Copyright 2010 <NAME>

    This file is part of KaVeikiaValdzia.lt website.

    PolicyFeed is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    PolicyFeed is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with PolicyFeed.  If not, see <http://www.gnu.org/licenses/>.
*/


// Requirements:
var Failures = require("PolicyFeed/Crawler/Failures");
var gluestick = require("gluestick");
var htmlunit = require("htmlunit");
var Queue = require("PolicyFeed/Crawler/Queue");
var organizations = require("KaVeikiaValdzia/tags/organizations");
var parser_utils = require("./utils");
var ringo_dates = require("ringo/utils/dates");

// Extends:
gluestick.extendModule(exports, "PolicyFeed/Crawler/Parser");
gluestick.extendModule(exports, "KaVeikiaValdzia/parsers/lrs.lt-common");

exports.disabled = false;

// Config:
exports.feed_url = [
    "http://www.lrs.lt/pls/proj/dokpaieska.rezult_l"
    ,"http://www.lrs.lt/pls/proj/dokpaieska.rezult_l?p_tr1=2&p_tr2=2&p_fix=y&p_gov=n&p_no=2"
    ,"http://www.lrs.lt/pls/proj/dokpaieska.rezult_l?p_tr1=2&p_tr2=2&p_fix=y&p_gov=n&p_no=3"
    ];


/**
 *
 */
exports.extractFeedItems = function(page) {
    this.validateFeedPage(page);
    htmlunit.fixPageUrls(page);

    var items = page.getByXPath('/html/body/div/table/tbody/tr[2]/td/table/tbody/tr/td/align/table[2]/tbody/tr').toArray();

    items.shift();
    if (items.length < 1) {
        return [];
    } else {
        return items.map(this.parseFeedItem()).filter(function(item) { return item });
    }
}


/**
 *
 */
exports.parseFeedItem = function(item) {
    var name = this.name;
    var doc_template = this.doc_template;
    var error = this.error;
    var orgmap = organizations.getNameMap();

    return function(item) {

        var result = {};
        var teises_aktas = {};

        var num = item.getFirstByXPath('./td[1]').asText();
        var content = item.getFirstByXPath('./td[2]');
        var published = item.getFirstByXPath('./td[3]').asText();
        var numeris = item.getFirstByXPath('./td[4]').asText();
        if (numeris)
            teises_aktas.numeris = numeris;
        var type = item.getFirstByXPath('./td[5]').asText();

        // url and title:
        var tspan = content.getFirstByXPath('./span');
        if (!tspan&& (num % 30 == 0)) {
            // Link is always missing for item #30 (might be a HtmlUnit bug):
            return false;
        }
        var tlink = tspan.getFirstByXPath('./a');
        var title = tlink.asText();
        var url = parser_utils.getCanonicalLrsUrl( tlink.getHrefAttribute().toString() );
        tspan.remove();

        var small_links = content.getByXPath('./small/a');
        if (small_links) {
            small_links = small_links.toArray();
            for each (var slink in small_links) {
                switch (slink.asText()) {
                    case "Priedai":
                        teises_aktas.priedai = slink.getHrefAttribute().toString();
                    break;
                    case "[Susiję dokumentai]":
                        teises_aktas.susije = slink.getHrefAttribute().toString();
                    break;
                    default:
                        Failures.write(url, { parser: name, url: url, data: {
                                error: "Unrecognized small link",
                                text: slink.asText(),
                                link: slink.getHrefAttribute().toString()
                                });
                }
                slink.remove();
            }
        }

        // get non-empty content lines:
        content = content.asText().split(/[\n;]/).filter(function(part) { return part.trim(); });

        var author = "";
        for each (var line in content) {
            line = line.split(/[:,\/]/).map(function(part) { return part.trim() });

            switch (line[0]) {
                case "Pateikė":
                case "Priėmė":
                case "Parengė":
                    if (line[1].slice(0, 12) == "Seimo narys " || line[1].slice(0, 11) == "Seimo narė ") {
{'fraction_non_alphanumeric': 0.13443628793689055, 'fraction_numerical': 0.008107811986413936, 'mean_word_length': 2.2243023666548924, 'pattern_counts': {'":': 10, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4886801', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2764, 'n_words': 634}