<commit_before>var URL = require('url'),
    request = require('request'),
    _ = require('underscore'),
    async = require('async'),
    cheerio = require('cheerio'),
    moment = require('moment'),
    mysql = require('mysql'),
    config = require('./config'),
    db = mysql.createConnection(config.database),
    columnMap = {
        // 'Name of the Candidate': 'candidate',
        'Filing Date': 'filing_date',
        // 'Name of Committee': 'committee',
        'Party Affiliation': 'party',
        'Office Sought': 'office'
    },
    columns = 'year committee candidate office party organization_date url'.split(' '),
    baseUrl = 'http://ocf.dc.gov/registration_statements/pcc/pcc_searchresult.asp?ftype=PCC',
    queue = async.queue(process),
    maxYear = +moment().format('YYYY') + 2,
    grandTotal = 0,
    recordsInserted = 0,
    recordsFound = 0;

db.connect(function (err) {
    if (err) {
        throw err;
    }
});

queue.push({year: 2014, page: 1});

function process(task, callback) {
    if (task.page) {
        processPage(task, callback);
    }
    else {
        insertRecord(task, callback);
    }
}

function processPage(task, callback) {
    var year = task.year,
        page = task.page,
        url = baseUrl + '&ele_year=' + year + '&whichpage=' + page,
        numPages, totalRecords;
    console.log('getting', url);
    request(url, function (err, response, body) {
        var m, $;
        if (err) {
            throw err;
        }
        if (body.match(/Your\s+search returned no results\./)) {
            console.log("  no records\n");
            numPages = totalRecords = 0;
        }
        else if (m = body.match(/Page (\d+) of (\d+)(?:&nbsp;)+Total Records : (\d+)/)) {
            if (numPages) {
                if (+m[1] !== page || +m[2] !== numPages || +m[3] !== totalRecords) {
                    throw 'Unexpected numbers (' + m[1] + ', ' + m[2] + ', ' + m[3] + ') for ' + url;
                }
            }
            else {
                numPages = +m[2];
                totalRecords = +m[3];
                console.log('%d pages, %d records', numPages, totalRecords);
            }
            if (page == 1) {
                grandTotal += totalRecords;
            }
            $ = cheerio.load(body);
            $('form[name=pcc_searchresult] table').children('[bgcolor]').each(function () {
                var $row = $(this),
                    r = {year: year},
                    $cells = $row.children(),
                    m;
                r.candidate = trim($cells.eq(0).text());
                r.committee = trim($cells.eq(1).text());
                r.url = URL.resolve(baseUrl, $cells.eq(0).find('a').attr('href'));
                if (m = r.url.match(/sys_cancom_id=(\d+)/)) {
                    r.id = +m[1];
                }
                recordsFound++;
                if (r.committee == 'N/A') {
                    r.committee = '[' + r.candidate.replace(/^([^,]+), ([^,]+)/, '$2 $1') + ' ' + year + ']';
                }
                console.log('getting', r.url);
                request(r.url, function (err, response, body) {
                    var $ = cheerio.load(body);
                    if (err) {
                        throw err;
                    }
                    $('tr').each(function () {
                        var subcells = $(this).find('td'),
                            label, value, column;
                        if (subcells.length != 2 ||
                            subcells.eq(0).attr('width') != '25%' ||
                            subcells.eq(1).attr('width') != '75%') {
                            return;
                        }
                        label = trim(subcells.eq(0).text());
                        value = trim(subcells.eq(1).text());
                        column = columnMap[label];
                        if (!column) {
                            return;
                        }
                        if (/_date$/.test(column)) {
                            value = moment(value).format('YYYY-MM-DD');
                        }
                        r[column] = value;
                    });
                    console.log('queuing', r);
                    queue.push(r, function (err) {
                        if (err) {
                            throw err;
                        }
                    });
                });
            });
        }
        else {
            throw 'Unexpected first page for ' + url + '\n' + body;
        }
        if (page < numPages) {
            page++;
        }
        else {
            if (recordsFound != totalRecords) {
                throw recordsFound + ' of total ' + totalRecords + ' records found';<commit_msg>skip empty values on detail page (to avoid overwriting filing date)<commit_after>var URL = require('url'),
    request = require('request'),
    _ = require('underscore'),
    async = require('async'),
    cheerio = require('cheerio'),
    moment = require('moment'),
    mysql = require('mysql'),
    config = require('./config'),
    db = mysql.createConnection(config.database),
    columnMap = {
        // 'Name of the Candidate': 'candidate',
        'Filing Date': 'filing_date',
        // 'Name of Committee': 'committee',
        'Party Affiliation': 'party',
        'Office Sought': 'office'
    },
    columns = 'year committee candidate office party organization_date url'.split(' '),
    baseUrl = 'http://ocf.dc.gov/registration_statements/pcc/pcc_searchresult.asp?ftype=PCC',
    queue = async.queue(process),
    maxYear = +moment().format('YYYY') + 2,
    grandTotal = 0,
    recordsInserted = 0,
    recordsFound = 0;

db.connect(function (err) {
    if (err) {
        throw err;
    }
});

queue.push({year: 2014, page: 1});

function process(task, callback) {
    if (task.page) {
        processPage(task, callback);
    }
    else {
        insertRecord(task, callback);
    }
}

function processPage(task, callback) {
    var year = task.year,
        page = task.page,
        url = baseUrl + '&ele_year=' + year + '&whichpage=' + page,
        numPages, totalRecords;
    console.log('getting', url);
    request(url, function (err, response, body) {
        var m, $;
        if (err) {
            throw err;
        }
        if (body.match(/Your\s+search returned no results\./)) {
            console.log("  no records\n");
            numPages = totalRecords = 0;
        }
        else if (m = body.match(/Page (\d+) of (\d+)(?:&nbsp;)+Total Records : (\d+)/)) {
            if (numPages) {
                if (+m[1] !== page || +m[2] !== numPages || +m[3] !== totalRecords) {
                    throw 'Unexpected numbers (' + m[1] + ', ' + m[2] + ', ' + m[3] + ') for ' + url;
                }
            }
            else {
                numPages = +m[2];
                totalRecords = +m[3];
                console.log('%d pages, %d records', numPages, totalRecords);
            }
            if (page == 1) {
                grandTotal += totalRecords;
            }
            $ = cheerio.load(body);
            $('form[name=pcc_searchresult] table').children('[bgcolor]').each(function () {
                var $row = $(this),
                    r = {year: year},
                    $cells = $row.children(),
                    m;
                r.candidate = trim($cells.eq(0).text());
                r.committee = trim($cells.eq(1).text());
                r.url = URL.resolve(baseUrl, $cells.eq(0).find('a').attr('href'));
                if (m = r.url.match(/sys_cancom_id=(\d+)/)) {
                    r.id = +m[1];
                }
                recordsFound++;
                if (r.committee == 'N/A') {
                    r.committee = '[' + r.candidate.replace(/^([^,]+), ([^,]+)/, '$2 $1') + ' ' + year + ']';
                }
                console.log('getting', r.url);
                request(r.url, function (err, response, body) {
                    var $ = cheerio.load(body);
                    if (err) {
                        throw err;
                    }
                    $('tr').each(function () {
                        var subcells = $(this).find('td'),
                            label, value, column;
                        if (subcells.length != 2 ||
                            subcells.eq(0).attr('width') != '25%' ||
                            subcells.eq(1).attr('width') != '75%') {
                            return;
                        }
                        label = trim(subcells.eq(0).text());
                        value = trim(subcells.eq(1).text());
                        column = columnMap[label];
                        if (!column || !value) {
                            return;
                        }
                        if (/_date$/.test(column)) {
                            value = moment(value).format('YYYY-MM-DD');
                        }
                        r[column] = value;
                    });
                    console.log('queuing', r);
                    queue.push(r, function (err) {
                        if (err) {
                            throw err;
                        }
                    });
                });
            });
        }
        else {
            throw 'Unexpected first page for ' + url + '\n' + body;
        }
        if (page < numPages) {
            page++;
        }
        else {
            if (recordsFound != totalRecords) {
                throw recordsFound + ' of total ' + totalRecords + ' records found';
{'fraction_non_alphanumeric': 0.15610010427528676, 'fraction_numerical': 0.007090719499478624, 'mean_word_length': 1.265768958185684, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '728122', 'n_tokens_mistral': 2767, 'n_tokens_neox': 2593, 'n_words': 548}