<commit_before>#!/usr/bin/env node
var fs = require('fs');
var path = require('path');

var async = require('async');
var ent = require('ent');
var funkit = require('funkit');
var request = require('request');
var cheerio = require('cheerio');
var osm = require('osmgeocoder');

require('date-utils');


var partial = funkit.partial;

module.exports = scrape;

function scrape(o, cb) {
    o = funkit.merge({
        output: 'data',
        silent: false
    }, o);
    var prefix = 'http://www.jyvaskylanseutu.fi/tapahtumat/';
    var out = o.silent? funkit.id: console.log;
    var begin = Date.today();
    var end = Date.tomorrow();

    getAmount(prefix + getLink(begin, end), function(err, amount) {
        if(err) return console.error(err);

        var links = getLinks(begin, end, amount);

        async.map(links, partial(scrapePage, prefix, out), function(err, d) {
            if(err) return cb(err);

            writeJSON(o.output, out, d, cb);
       });
    });
}

function getLinks(begin, end, amount) {
    // there are 20 items per each page, this generates those links
    return funkit.range(0, amount, 20).map(function(k) {
        return getLink(begin, end, k);
    });
}

function getLink(begin, end, limit) {
    limit = limit || 0;

    function formatBegin(d) {
        return '' + d.getFullYear() + funkit.zfill(2, d.getMonth() + 1) + funkit.zfill(2, d.getDate());
    }

    function formatEnd(d) {
        return '' + d.getDate() + '.' + funkit.zfill(2, d.getMonth() + 1) + '.' + funkit.zfill(2, d.getFullYear());
    }

    return 'main.php?date=' + formatBegin(begin) + '&hPvm_loppu=' + formatEnd(end) + '&rajaa=' + limit + '&hLuokka[]=0';
}

function getAmount(url, cb) {
    request(url, function(req, res, data) {
        var $ = cheerio.load(data);
        var tdText = $($('td')[0]).html();
        var c = separate(tdText, 'Tulokset', '<br>').split('/')[1].trim();

        cb(null, parseInt(c, 10));
    });
}

function scrapePage(prefix, out, url, cb) {
    var target = prefix + url;

    request(prefix + url, function(req, res, data) {
        var $ = cheerio.load(data);

        var eventLinks = $('.tapahtumatiedot a').map(function(i, e) {
            return prefix + $(e).attr('href');
        });

        async.map(eventLinks, function(u, cb) {
            request(u, function(req, res, data) {
                var $ = cheerio.load(data);

                var info = ent.decode($($('.tapahtumatiedot')[0]).html());

                if(info) {
                    var d = {
                        name: $($('.tapahtumaotsikko')[0]).text()
                    };

                    async.each(Object.keys(parsers), function(k, cb) {
                        parsers[k](info, function(err, v) {
                            if(err) return cb(err);

                            d[k] = v;

                            cb();
                        });
                    }, function(err) {
                        if(err) return console.error(err);

                        cb(null, d);
                    });
                }
                else cb();
            });
        }, cb);
    });
}

var separate = funkit.separate;
var ltrim = funkit.ltrim;
var rtrim = funkit.rtrim;

var parsers = {<commit_msg>Attach ids to the scraped data<commit_after>#!/usr/bin/env node
var fs = require('fs');
var path = require('path');
var qs = require('querystring');

var async = require('async');
var ent = require('ent');
var funkit = require('funkit');
var request = require('request');
var cheerio = require('cheerio');
var osm = require('osmgeocoder');

require('date-utils');


var partial = funkit.partial;

module.exports = scrape;

function scrape(o, cb) {
    o = funkit.merge({
        output: 'data',
        silent: false
    }, o);
    var prefix = 'http://www.jyvaskylanseutu.fi/tapahtumat/';
    var out = o.silent? funkit.id: console.log;
    var begin = Date.today();
    var end = Date.tomorrow();

    getAmount(prefix + getLink(begin, end), function(err, amount) {
        if(err) return console.error(err);

        var links = getLinks(begin, end, amount);

        async.map(links, partial(scrapePage, prefix, out), function(err, d) {
            if(err) return cb(err);

            writeJSON(o.output, out, d, cb);
       });
    });
}

function getLinks(begin, end, amount) {
    // there are 20 items per each page, this generates those links
    return funkit.range(0, amount, 20).map(function(k) {
        return getLink(begin, end, k);
    });
}

function getLink(begin, end, limit) {
    limit = limit || 0;

    function formatBegin(d) {
        return '' + d.getFullYear() + funkit.zfill(2, d.getMonth() + 1) + funkit.zfill(2, d.getDate());
    }

    function formatEnd(d) {
        return '' + d.getDate() + '.' + funkit.zfill(2, d.getMonth() + 1) + '.' + funkit.zfill(2, d.getFullYear());
    }

    return 'main.php?date=' + formatBegin(begin) + '&hPvm_loppu=' + formatEnd(end) + '&rajaa=' + limit + '&hLuokka[]=0';
}

function getAmount(url, cb) {
    request(url, function(req, res, data) {
        var $ = cheerio.load(data);
        var tdText = $($('td')[0]).html();
        var c = separate(tdText, 'Tulokset', '<br>').split('/')[1].trim();

        cb(null, parseInt(c, 10));
    });
}

function scrapePage(prefix, out, url, cb) {
    var target = prefix + url;

    request(prefix + url, function(req, res, data) {
        var $ = cheerio.load(data);

        var eventLinks = $('.tapahtumatiedot a').map(function(i, e) {
            return prefix + $(e).attr('href');
        });

        async.map(eventLinks, function(u, cb) {
            request(u, function(req, res, data) {
                var $ = cheerio.load(data);

                var info = ent.decode($($('.tapahtumatiedot')[0]).html());

                if(info) {
                    var d = {
                        name: $($('.tapahtumaotsikko')[0]).text(),
                        id: qs.parse(u.split('?')[1]).id
                    };

                    async.each(Object.keys(parsers), function(k, cb) {
                        parsers[k](info, function(err, v) {
                            if(err) return cb(err);

                            d[k] = v;

                            cb();
                        });
                    }, function(err) {
                        if(err) return console.error(err);

                        cb(null, d);
                    });
                }
                else cb();
            });
        }, cb);
    });
}

var separate = funkit.separate;
var ltrim = funkit.ltrim;
var rtrim = funkit.rtrim;

var parsers = {
{'fraction_non_alphanumeric': 0.16903108964684577, 'fraction_numerical': 0.005885904014488379, 'mean_word_length': 1.9704168534289557, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5235084', 'n_tokens_mistral': 2213, 'n_tokens_neox': 2102, 'n_words': 472}