<commit_before>#!/usr/bin/env node

const url = require('url');
const cheerio = require('cheerio');
const _ = require('lodash');
const csvStringify = require('csv-stringify/lib/sync');
const request = require('./request');
const ancHomeUrl = 'https://anc.dc.gov/';

request(ancHomeUrl).then(html => cheerio.load(html))
    .then(getAncData)
    .then(function (data) {
        console.log(csvStringify(
            data,
            {
                header: true,
                columns: [
                    'smd',
                    'last_name',
                    'first_name',
                    'suffix',
                    'chair',
                    'address',
                    'zip',
                    'phone',
                    'email',
                ],
            }
        ));
    });

async function getAncData($) {
    const links = $('ul.menu-sub > li.leaf > a').get();
    const data = [];
    for (const a of links) {
        const $a = $(a);
        let m = $a.text().trim().match(/^ANC\s+[1-8][A-G]$/);
        if (!m) {
            continue;
        }
        const ancUrl = url.resolve(ancHomeUrl, $a.attr('href'));
        $ = await getCheerio(ancUrl);
        const headers = $('table > thead > tr > th')
            .map((i, th) => $(th).text().trim().toLowerCase())
            .get();
        const rows = $('table > tbody > tr').get();
        for (const row of rows) {
            const values = $(row).find('td')
                .map((i, th) => $(th).text().trim())
                .get();
            const record = _.zipObject(headers, values);
            if (!record.name) {
                continue;
            }
            m = record.name.match(/^(.+?)\s+Chairperson$/);
            if (m) {
                record.name = m[1];
                record.chair = 'Y';
            }
            else {
                record.chair = '';
            }
            m = record.name.match(/^\s*(\S.*?)\s+(\S+?)(?:,? (Jr|Sr|I+V?)\.?)?\s*$/);
            if (m) {
                record.first_name = m[1];
                record.last_name = m[2];
                record.suffix = m[3];
            }
            else {
                record.last_name = record.name;
                record.first_name = '';
                record.suffix = '';
            }
            delete record.name;
            m = record.address.match(/^(.*?)\s*Washington,?\s*(?:DC\s*)?(\d+)$/);
            if (m) {
                record.address = m[1];
                record.zip = m[2];
            }
            else {
                record.zip = '';
            }
            data.push(record);
        }
    }
    return data;
}

function getCheerio(requestOptions) {
    return request(requestOptions)
        .then(html => cheerio.load(html));
}<commit_msg>avoid duplicates in ANC data (3G04 is in 2 wards)<commit_after>#!/usr/bin/env node

const url = require('url');
const cheerio = require('cheerio');
const _ = require('lodash');
const csvStringify = require('csv-stringify/lib/sync');
const request = require('./request');
const ancHomeUrl = 'https://anc.dc.gov/';

request(ancHomeUrl).then(html => cheerio.load(html))
    .then(getAncData)
    .then(function (data) {
        console.log(csvStringify(
            Object.values(data),
            {
                header: true,
                columns: [
                    'smd',
                    'last_name',
                    'first_name',
                    'suffix',
                    'chair',
                    'address',
                    'zip',
                    'phone',
                    'email',
                ],
            }
        ));
    });

async function getAncData($) {
    const links = $('ul.menu-sub > li.leaf > a').get();
    const data = [];
    for (const a of links) {
        const $a = $(a);
        let m = $a.text().trim().match(/^ANC\s+[1-8][A-G]$/);
        if (!m) {
            continue;
        }
        const ancUrl = url.resolve(ancHomeUrl, $a.attr('href'));
        $ = await getCheerio(ancUrl);
        const headers = $('table > thead > tr > th')
            .map((i, th) => $(th).text().trim().toLowerCase())
            .get();
        const rows = $('table > tbody > tr').get();
        for (const row of rows) {
            const values = $(row).find('td')
                .map((i, th) => $(th).text().trim())
                .get();
            const record = _.zipObject(headers, values);
            if (!record.name) {
                continue;
            }
            m = record.name.match(/^(.+?)\s+Chairperson$/);
            if (m) {
                record.name = m[1];
                record.chair = 'Y';
            }
            else {
                record.chair = '';
            }
            m = record.name.match(/^\s*(\S.*?)\s+(\S+?)(?:,? (Jr|Sr|I+V?)\.?)?\s*$/);
            if (m) {
                record.first_name = m[1];
                record.last_name = m[2];
                record.suffix = m[3];
            }
            else {
                record.last_name = record.name;
                record.first_name = '';
                record.suffix = '';
            }
            delete record.name;
            m = record.address.match(/^(.*?)\s*Washington,?\s*(?:DC\s*)?(\d+)$/);
            if (m) {
                record.address = m[1];
                record.zip = m[2];
            }
            else {
                record.zip = '';
            }
            data[record.smd] = record;
        }
    }
    return data;
}

function getCheerio(requestOptions) {
    return request(requestOptions)
        .then(html => cheerio.load(html));
}
{'fraction_non_alphanumeric': 0.17167229429536165, 'fraction_numerical': 0.003554291807357384, 'mean_word_length': 1.2342199285430726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2679692', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1633, 'n_words': 308}