<commit_before>#!/usr/bin/env node
/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4 fileencoding=utf-8 : */
/*
 *     Copyright 2013 <NAME>
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */

// enable JavaScript strict mode.
"use strict";

var fs = require("fs");
var http = require("http");
var path = require("path");
var url = require("url");
var util = require("util");
var agentdb = require("./lib/samplx-agentdb.js");
var HashTable = require("./lib/hash.js").HashTable;

// enable debug messages
var debug = false;

/** size of the hash table. can use test/tune-hash.js to compute. */
var HASH_TABLE_SIZE = 4339;

/**
 *  Download JSON object from a remote URL.
 *
 *  @param path to request.
 *  @param done callback.
 */
function getJSON(jsonUrl, done) {
    var parsedUrl = url.parse(jsonUrl);
    
    var options = {
        "host" : parsedUrl.hostname,
        "hostname": parsedUrl.hostname,
        "port" : parsedUrl.port || 80,
        "path" : parsedUrl.pathname,
        "method" : "GET",
        // "agent" : false,
    };
    
    debug && console.error("prepublish:getJSON('" + jsonUrl + "', done)");
    debug && console.error("options=" + util.inspect(options));
    var req = http.get(options, function (res) {
        debug && console.error("STATUS: " + res.statusCode);
        debug && console.error("HEADERS: " + JSON.stringify(res.headers));
        res.setEncoding('utf8');
        var data = '';
        res.on('data', function (chunk) {
            debug && console.error('getJSON: chunk="' + chunk + '"');
            data += chunk;
        });
        res.on('end', function() {
            debug && console.error('getJSON: end of request.');
            done(JSON.parse(data));
        });
    });

    req.on('error', function(error) {
        console.error("ERROR: " + error.message);
    });
}

/**
 *  Main entry point for prepublish script.
 */
function main() {

    var database = fs.createWriteStream('./lib/database-min.js', { flags: 'w', encoding: 'utf8' });
    var testFile;

    database.write("// Do not edit this file, your changes will be lost.\n");
    database.write("module.exports = { \n");
    getJSON('http://alscan.info/agent_groups/groups.json', function (groups) {
        var s = JSON.stringify(groups);
        database.write("groups:" + s + ",\n");
        testFile = fs.createWriteStream('./test/groups.json', { flags: 'w', encoding: 'utf8' });
        testFile.write(s+"\n");
        testFile.end();
        getJSON('http://alscan.info/agent_sources/sources.json', function (sources) {
            var s = JSON.stringify(sources);
            database.write("sources:" + s + ",\n");
            testFile = fs.createWriteStream('./test/sources.json', { flags: 'w', encoding: 'utf8' });
            testFile.write(s+"\n");
            testFile.end();
            getJSON('http://alscan.info/patterns/patterns.json', function (patterns) {
                database.write("patterns:[\n");
                patterns.forEach(function (pattern) {
                    database.write("{p: new RegExp('" + pattern.pattern + "'), x:" + 
                            agentdb.getX(pattern.groupId, pattern.sourceId) + "},");
                });
                database.write("],\n");
                testFile = fs.createWriteStream('./test/patterns.json', { flags: 'w', encoding: 'utf8' });
                testFile.write(JSON.stringify(patterns)+"\n");
                testFile.end();
                getJSON('http://alscan.info/agents/agents.json', function (agents) {
                    var hash = new HashTable(HASH_TABLE_SIZE);
                    agents.forEach(function (agent) {
                        if (agent.status < 2) {
                            var obj = { a: agent.agent, x: agentdb.getX(agent.groupId, agent.sourceId) };
                            hash.add('a', obj);
                        }
                    });
                    database.write("table:" + JSON.stringify(hash.table) + ",\n");
                    database.write("HASH_TABLE_SIZE:" + HASH_TABLE_SIZE + ",\n");
                    database.write("hash: undefined\n}\n");
                    testFile = fs.createWriteStream('./test/agents.json', { flags: 'w', encoding: 'utf8' });
                    testFile.write(JSON.stringify(agents)+"\n");
                    testFile.end();
                    if (debug) {
                        var history = hash.getHistory();
                        console.log("Hash table history");
                        for (var n=0; n < history.length; n++) {
                            console.log("  Depth=" + n + ", count=" + history[n] + ", percent=" + (history[n] / HASH_TABLE_SIZE));
                        }
                    }
                });
            });
        });
    });
}

main();<commit_msg>Handle size, seed and multiplier hash table parameters. Move data files to data/ directory.<commit_after>#!/usr/bin/env node
/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4 fileencoding=utf-8 : */
/*
 *     Copyright 2013 <NAME>
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */

// enable JavaScript strict mode.
"use strict";

var fs = require("fs");
var http = require("http");
var path = require("path");
var url = require("url");
var util = require("util");
var agentdb = require("./lib/samplx-agentdb.js");
var HashTable = require("./lib/hash.js").HashTable;

// enable debug messages
var debug = false;

/** size of the hash table. can use utils/tune-hash.js to compute. */
var HASH_TABLE_SIZE = 3853;

/** hash table seed value. */
var HASH_TABLE_SEED = 3884;

/** hash table multiplier. */
var HASH_TABLE_MULTIPLIER = 33;

/**
 *  Download JSON object from a remote URL.
 *
 *  @param path to request.
 *  @param done callback.
 */
function getJSON(jsonUrl, done) {
    var parsedUrl = url.parse(jsonUrl);
    
    var options = {
        "host" : parsedUrl.hostname,
        "hostname": parsedUrl.hostname,
        "port" : parsedUrl.port || 80,
        "path" : parsedUrl.pathname,
        "method" : "GET",
        // "agent" : false,
    };
    
    debug && console.error("prepublish:getJSON('" + jsonUrl + "', done)");
    debug && console.error("options=" + util.inspect(options));
    var req = http.get(options, function (res) {
        debug && console.error("STATUS: " + res.statusCode);
        debug && console.error("HEADERS: " + JSON.stringify(res.headers));
        res.setEncoding('utf8');
        var data = '';
        res.on('data', function (chunk) {
            debug && console.error('getJSON: chunk="' + chunk + '"');
            data += chunk;
        });
        res.on('end', function() {
            debug && console.error('getJSON: end of request.');
            done(JSON.parse(data));
        });
    });

    req.on('error', function(error) {
        console.error("ERROR: " + error.message);
    });
}

/**
 *  Main entry point for prepublish script.
 */
function main() {

    var database = fs.createWriteStream('./lib/database-min.js', { flags: 'w', encoding: 'utf8' });
    var testFile;

    fs.mkdir('./data', 493, function (error) {  // 493 == 0755 (no octal in strict mode.)
        // ignore errors.
    });
    
    database.write("// Do not edit this file, your changes will be lost.\n");
    database.write("module.exports = { \n");
    getJSON('http://alscan.info/agent_groups/groups.json', function (groups) {
        var s = JSON.stringify(groups);
        database.write("groups:" + s + ",\n");
        testFile = fs.createWriteStream('./data/groups.json', { flags: 'w', encoding: 'utf8' });
        testFile.write(s+"\n");
        testFile.end();
        getJSON('http://alscan.info/agent_sources/sources.json', function (sources) {
            var s = JSON.stringify(sources);
            database.write("sources:" + s + ",\n");
            testFile = fs.createWriteStream('./data/sources.json', { flags: 'w', encoding: 'utf8' });
            testFile.write(s+"\n");
            testFile.end();
            getJSON('http://alscan.info/patterns/patterns.json', function (patterns) {
                database.write("patterns:[\n");
                patterns.forEach(function (pattern) {
                    database.write("{p: new RegExp('" + pattern.pattern + "'), x:" + 
                            agentdb.getX(pattern.groupId, pattern.sourceId) + "},");
                });
                database.write("],\n");
                testFile = fs.createWriteStream('./data/patterns.json', { flags: 'w', encoding: 'utf8' });
                testFile.write(JSON.stringify(patterns)+"\n");
                testFile.end();
                getJSON('http://alscan.info/agents/agents.json', function (agents) {
                    var hash = new HashTable(HASH_TABLE_SIZE, HASH_TABLE_SEED, HASH_TABLE_MULTIPLIER);
                    agents.forEach(function (agent) {
                        if (agent.status < 2) {
                            var obj = { a: agent.agent, x: agentdb.getX(agent.groupId, agent.sourceId) };
                            hash.add('a', obj);
                        }
                    });
                    database.write("table:" + JSON.stringify(hash.table) + ",\n");
                    database.write("HASH_TABLE_SIZE:" + HASH_TABLE_SIZE + ",\n");
                    database.write("HASH_TABLE_SEED:" + HASH_TABLE_SEED + ",\n");
                    database.write("HASH_TABLE_MULTIPLIER:" + HASH_TABLE_MULTIPLIER + ",\n");
                    database.write("hash: undefined\n}\n");
                    testFile = fs.createWriteStream('./data/agents.json', { flags: 'w', encoding: 'utf8' });
                    testFile.write(JSON.stringify(agents)+"\n");
                    testFile.end();
                    if (debug) {
                        var history = hash.getHistory();
                        console.log("Hash table history");
                        for (var n=0; n < history.length; n++) {
                            console.log("  Depth=" + n + ", count=" + history[n] + ", percent=" + (history[n] / HASH_TABLE_SIZE));
                        }
                    }
                });
            });
        });
    });
}

main();
{'fraction_non_alphanumeric': 0.15577264653641207, 'fraction_numerical': 0.006039076376554174, 'mean_word_length': 2.0101577118417535, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2302011', 'n_tokens_mistral': 3276, 'n_tokens_neox': 3167, 'n_words': 764}