<commit_before>/*
    jcadmin.js  -  by <NAME>

    https://github.com/cosinekitty/jcadmin
*/
var path = require('path');
var fs = require('fs');
var express = require('express');
var app = express();
var logprefix = require('log-prefix');

function ZeroPad(n, d) {
    var s = '' + n;
    while (s.length < d) {
        s = '0' + s;
    }
    return s;
}

var DaysOfWeek = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];

logprefix(function(){
    var now = new Date();
    var text = '[' + now.getFullYear();
    text += '-' + ZeroPad(now.getMonth() + 1, 2);
    text += '-' + ZeroPad(now.getDate(), 2);
    text += ' ' + ZeroPad(now.getHours(), 2);
    text += ':' + ZeroPad(now.getMinutes(), 2);
    text += ':' + ZeroPad(now.getSeconds(), 2);
    text += '.' + ZeroPad(now.getMilliseconds(), 3);
    text += ' ' + DaysOfWeek[now.getDay()];
    text += '] %s';
    return text;
});

// Parse the command line for configuration parameters.
// Usage:  node jcadmin.js port path
var port = 9393;
var jcpath = '.';

if (process.argv.length > 2) {
    port = parseInt(process.argv[2]);
}

if (process.argv.length > 3) {
    jcpath = process.argv[3];
}

var jcLogFile = ValidateFileExists(path.join(jcpath, 'callerID.dat'));
var whiteListFileName = ValidateFileExists(path.join(jcpath, 'whitelist.dat'));
var blackListFileName = ValidateFileExists(path.join(jcpath, 'blacklist.dat'));
console.log('Monitoring jcblock path %s', jcpath);

app.use(express.static('public'));

app.get('/', (request, response) => {
    response.sendFile(path.join(__dirname, 'index.html'));
});

function ValidateFileExists(filename) {
    try {
        fs.statSync(filename);
        return filename;
    } catch (e) {
        console.log('FATAL ERROR: file does not exist: %s', filename);
        console.log('Try adjusting the path passed on the command line.');
        process.exit(1);
    }
}

function MakeDateTimeString(date, time) {
    // date = '011916', time = '1623'  ==>  '2016-01-19 16:23'
    var year = (2000 + parseInt(date.substr(4, 2), 10)) + '';
    return year + '-' + date.substr(0, 2) + '-' + date.substr(2, 2) + ' ' + time.substr(0, 2) + ':' + time.substr(2, 2);
}

function FilterNameNumber(text) {
    text = text.trim();
    if (text == "O") {
        return "";
    }
    return text;
}

function ParseCallLine(line) {
    // Examples of caller ID data:
    // B-DATE = 011916--TIME = 1616--NMBR = 8774845967--NAME = TOLL FREE CALLE--
    // --DATE = 011916--TIME = 1623--NMBR = O--NAME = O--
    var m = line.match(/([WB\-])-DATE = (\d{6})--TIME = (\d{4})--NMBR = ([^\-]*)--NAME = ([^\-]*)--/);
    if (m) {
        return {
            'status':   m[1],     // W, B, -; whitelisted, blocked, neither
            'when':     MakeDateTimeString(m[2], m[3]),
            'number':   FilterNameNumber(m[4]),
            'name':     FilterNameNumber(m[5])
        };
    }
    return null;
}

function ParseRecentCalls(text, start, limit) {
    var lines = text.split('\n');
    var calls = [];
    var total = 0;
    for (var i = lines.length - 1; i >= 0; --i) {
        var c = ParseCallLine(lines[i]);
        if (c) {
            if (total >= start && calls.length < limit) {
                calls.push(c);
            }
            ++total;
        }
    }

    return {
        'total': total,
        'start': start,
        'limit': limit,
        'calls': calls
    };
}

function ParseIntParam(text, fallback) {
    var value = parseInt(text);
    if (isNaN(value)) {
        return fallback;
    }
    return value;
}

app.get('/api/poll', (request, response) => {
    // https://nodejs.org/api/fs.html#fs_fs_stat_path_callback
    // https://nodejs.org/api/fs.html#fs_class_fs_stats
    // http://stackoverflow.com/questions/7559555/last-modified-file-date-in-node-js

    response.type('json');
    var reply = {};

    // Start 3 async requests. The the first one to encounter an error
    // or the last one to succeed ends the response for us.

    function StatCallback(err, stats, reply, field) {
        if (err) {
            reply = { 'error':err, 'field':field };
            response.end(JSON.stringify(reply));
        } else {
            if (!reply.error) {
                reply[field] = { 'modified' : stats.mtime };
                if (reply.callerid && reply.whitelist && reply.blacklist) {
                    response.end(JSON.stringify(reply));
                }
            }
        }
    }

    fs.stat(jcLogFile,         (err, stats) => StatCallback(err, stats, reply, 'callerid' ));
    fs.stat(whiteListFileName, (err, stats) => StatCallback(err, stats, reply, 'whitelist'));
    fs.stat(blackListFileName, (err, stats) => StatCallback(err, stats, reply, 'blacklist'));
});

app.get('/api/calls/:start/:limit', (request, response) => {
    var start = ParseIntParam(request.params.start, 0);
    var limit = ParseIntParam(request.params.limit, 1000000000);
    fs.readFile(jcLogFile, 'utf8', (err, data) => {
        var replyJson = err ? { 'error' : err } : ParseRecentCalls(data, start, limit);
        response.type('json');
        response.end(JSON.stringify(replyJson));
    });
});

function PhoneListContainsNumber(data, number) {
    var lines = data.split('\n');
    for (var i=0; i < lines.length; ++i) {
        if (!lines[i].startsWith('#')) {
            var limit = lines[i].indexOf('?');
            if (limit < 0) limit = 19;
            var entry = lines[i].substr(0, limit).trim();
            if (number === entry) {
                return true;
            }
        }
    }
    return false;
}

app.get('/api/number/:phonenumber', (request, response) => {
    response.type('json');
    var phonenumber = request.params.phonenumber;
    fs.readFile(whiteListFileName, 'utf8', (werr, wdata) => {
        if (werr) {
            response.end(JSON.stringify({'error' : werr}));
        } else {
            // Search whitelist for phone number.
            // If found, return immediately that the number is whitelisted.
            if (PhoneListContainsNumber(wdata, phonenumber)) {
                response.end(JSON.stringify({'status' : 'W'}));
            } else {
                // Search blacklist for the phone number.
                // If found, the phone number is blocked, otherwise it is neither
                // whitelisted nor blocked.
                fs.readFile(blackListFileName, 'utf8', (berr, bdata) => {
                    if (berr) {
                        response.end(JSON.stringify({'error': berr}));
                    } else {
                        var blacklisted = PhoneListContainsNumber(bdata, phonenumber);
                        response.end(JSON.stringify({
                            'status' : blacklisted ? 'B' : '-'
                        }));
                    }
                });
            }
        }
    });
});<commit_msg>Added /api/fetch method to download whitelist or blacklist data.<commit_after>/*
    jcadmin.js  -  by <NAME>

    https://github.com/cosinekitty/jcadmin
*/
var path = require('path');
var fs = require('fs');
var express = require('express');
var app = express();
var logprefix = require('log-prefix');

function ZeroPad(n, d) {
    var s = '' + n;
    while (s.length < d) {
        s = '0' + s;
    }
    return s;
}

var DaysOfWeek = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];

logprefix(function(){
    var now = new Date();
    var text = '[' + now.getFullYear();
    text += '-' + ZeroPad(now.getMonth() + 1, 2);
    text += '-' + ZeroPad(now.getDate(), 2);
    text += ' ' + ZeroPad(now.getHours(), 2);
    text += ':' + ZeroPad(now.getMinutes(), 2);
    text += ':' + ZeroPad(now.getSeconds(), 2);
    text += '.' + ZeroPad(now.getMilliseconds(), 3);
    text += ' ' + DaysOfWeek[now.getDay()];
    text += '] %s';
    return text;
});

// Parse the command line for configuration parameters.
// Usage:  node jcadmin.js port path
var port = 9393;
var jcpath = '.';

if (process.argv.length > 2) {
    port = parseInt(process.argv[2]);
}

if (process.argv.length > 3) {
    jcpath = process.argv[3];
}

var jcLogFile = ValidateFileExists(path.join(jcpath, 'callerID.dat'));
var whiteListFileName = ValidateFileExists(path.join(jcpath, 'whitelist.dat'));
var blackListFileName = ValidateFileExists(path.join(jcpath, 'blacklist.dat'));
console.log('Monitoring jcblock path %s', jcpath);

app.use(express.static('public'));

app.get('/', (request, response) => {
    response.sendFile(path.join(__dirname, 'index.html'));
});

function ValidateFileExists(filename) {
    try {
        fs.statSync(filename);
        return filename;
    } catch (e) {
        console.log('FATAL ERROR: file does not exist: %s', filename);
        console.log('Try adjusting the path passed on the command line.');
        process.exit(1);
    }
}

function MakeDateTimeString(date, time) {
    // date = '011916', time = '1623'  ==>  '2016-01-19 16:23'
    var year = (2000 + parseInt(date.substr(4, 2), 10)) + '';
    return year + '-' + date.substr(0, 2) + '-' + date.substr(2, 2) + ' ' + time.substr(0, 2) + ':' + time.substr(2, 2);
}

function FilterNameNumber(text) {
    text = text.trim();
    if (text == "O") {
        return "";
    }
    return text;
}

function ParseCallLine(line) {
    // Examples of caller ID data:
    // B-DATE = 011916--TIME = 1616--NMBR = 8774845967--NAME = TOLL FREE CALLE--
    // --DATE = 011916--TIME = 1623--NMBR = O--NAME = O--
    var m = line.match(/([WB\-])-DATE = (\d{6})--TIME = (\d{4})--NMBR = ([^\-]*)--NAME = ([^\-]*)--/);
    if (m) {
        return {
            'status':   m[1],     // W, B, -; whitelisted, blocked, neither
            'when':     MakeDateTimeString(m[2], m[3]),
            'number':   FilterNameNumber(m[4]),
            'name':     FilterNameNumber(m[5])
        };
    }
    return null;
}

function ParseRecentCalls(text, start, limit) {
    var lines = text.split('\n');
    var calls = [];
    var total = 0;
    for (var i = lines.length - 1; i >= 0; --i) {
        var c = ParseCallLine(lines[i]);
        if (c) {
            if (total >= start && calls.length < limit) {
                calls.push(c);
            }
            ++total;
        }
    }

    return {
        'total': total,
        'start': start,
        'limit': limit,
        'calls': calls
    };
}

function ParseIntParam(text, fallback) {
    var value = parseInt(text);
    if (isNaN(value)) {
        return fallback;
    }
    return value;
}

app.get('/api/poll', (request, response) => {
    // https://nodejs.org/api/fs.html#fs_fs_stat_path_callback
    // https://nodejs.org/api/fs.html#fs_class_fs_stats
    // http://stackoverflow.com/questions/7559555/last-modified-file-date-in-node-js

    response.type('json');
    var reply = {};

    // Start 3 async requests. The the first one to encounter an error
    // or the last one to succeed ends the response for us.

    function StatCallback(err, stats, reply, field) {
        if (err) {
            reply = { 'error':err, 'field':field };
            response.end(JSON.stringify(reply));
        } else {
            if (!reply.error) {
                reply[field] = { 'modified' : stats.mtime };
                if (reply.callerid && reply.whitelist && reply.blacklist) {
                    response.end(JSON.stringify(reply));
                }
            }
        }
    }

    fs.stat(jcLogFile,         (err, stats) => StatCallback(err, stats, reply, 'callerid' ));
    fs.stat(whiteListFileName, (err, stats) => StatCallback(err, stats, reply, 'whitelist'));
    fs.stat(blackListFileName, (err, stats) => StatCallback(err, stats, reply, 'blacklist'));
});

app.get('/api/calls/:start/:limit', (request, response) => {
    var start = ParseIntParam(request.params.start, 0);
    var limit = ParseIntParam(request.params.limit, 1000000000);
    fs.readFile(jcLogFile, 'utf8', (err, data) => {
        var replyJson = err ? { 'error' : err } : ParseRecentCalls(data, start, limit);
        response.type('json');
        response.end(JSON.stringify(replyJson));
    });
});

function PhoneListContainsNumber(data, number) {
    var lines = data.split('\n');
    for (var i=0; i < lines.length; ++i) {
        if (!lines[i].startsWith('#')) {
            var limit = lines[i].indexOf('?');
            if (limit < 0) limit = 19;
            var entry = lines[i].substr(0, limit).trim();
            if (number === entry) {
                return true;
            }
        }
    }
    return false;
}

app.get('/api/fetch/:filetype', (request, response) => {
    response.type('json');

    var filename;
    switch (request.params.filetype) {
        case 'whitelist':  filename = whiteListFileName;  break;
        case 'blacklist':  filename = blackListFileName;  break;
        default:
            response.end(JSON.stringify({'error' : 'Invalid filetype ' + request.params.filetype}));
            return;
    }

    fs.readFile(filename, 'utf8', (err, data) => {
        if (err) {
            response.end(JSON.stringify({'error' : err}));
        } else {
            reply = { 'table' : {} };
            var lines = data.split('\n');
            for (var line of lines) {
                if (!line.startsWith('#') && (line.length >= 25)) {
                    var limit = line.indexOf('?');
                    if (limit < 0) limit = 19;
                    var pattern = line.substr(0, limit).trim();
                    var comment = line.substr(25).trim();
                    reply.table[pattern] = comment;
                }
            }
            response.end(JSON.stringify(reply));
        }
    });
});

app.get('/api/number/:phonenumber', (request, response) => {
    response.type('json');
    var phonenumber = request.params.phonenumber;
    fs.readFile(whiteListFileName, 'utf8', (werr, wdata) => {
        if (werr) {
            response.end(JSON.stringify({'error' : werr}));
        } else {
            // Search whitelist for phone number.
            // If found, return immediately that the number is whitelisted.
            if (PhoneListContainsNumber(wdata, phonenumber)) {
                response.end(JSON.stringify({'status' : 'W'}));
            } else {
                // Search blacklist for the phone number.
                // If found, the phone number is blocked, otherwise it is neither
                // whitelisted nor blocked.
                fs.readFile(blackListFileName, 'utf8', (berr, bdata) => {
                    if (berr) {
                        response.end(JSON.stringify({'error': berr}));
                    } else {
                        var blacklisted = PhoneListContainsNumber(bdata, phonenumber);
                        response.end(JSON.stringify({
                            'status' : blacklisted ? 'B' : '-'
                        }));
                    }
                });
            }
        }
    });
});
{'fraction_non_alphanumeric': 0.17025427149199515, 'fraction_numerical': 0.017018700390152026, 'mean_word_length': 2.0972916666666666, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 38, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6043399', 'n_tokens_mistral': 4915, 'n_tokens_neox': 4695, 'n_words': 1053}