<commit_before>#!/usr/bin/env node

/*
 * Allow you smoothly surf on many websites blocking non-mainland visitors.
 * Copyright (C) 2012, 2013 <NAME> http://zhuzhu.org
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */


var util = require('util');
var http = require('http');
http.globalAgent.maxSockets = Infinity;
var cluster = require('cluster');

var colors = require('colors');
var argv = require('optimist')
    .default('port', 8888)
    .boolean('local_only')
    .boolean('production')
    .argv
;

var sogou = require('../shared/sogou');
var shared_tools = require('../shared/tools');
var server_utils = require('./utils');


var local_addr, local_port, proxy_addr;
if (!argv.production) {
    local_port = argv.port;
    if (argv.local_only) {
        local_addr = '127.0.0.1';
        proxy_addr = '127.0.0.1:' + local_port;
    } else {
        local_addr = '0.0.0.0';
        proxy_addr = server_utils.get_first_external_ip() + ':' + local_port;
    }
} else {
    local_port = process.env.PORT || 8888;
    local_addr = '0.0.0.0';
    proxy_addr = 'proxy.uku.im:80';
}
var pac_file_content = shared_tools.url2pac(require('../shared/urls').url_list, proxy_addr);


// what are the life cycles of variables in nodejs?
var sogou_server_addr;
var reset_count = 0, MAX_RESET_COUNT = 1;
var timeout_count = 0, MAX_TIMEOUT_COUNT = 4;
var in_changing_server = false, last_error_code = null;
function change_sogou_server(error_code) {
    if (true === in_changing_server) {
        return;  // should already be in the process of changing new server
    }
    in_changing_server = true;

    if ('string' === typeof error_code) {
        last_error_code = error_code;
    } else {
        last_error_code = null;
    }
    server_utils.renew_sogou_server(function(new_addr) {
        sogou_server_addr = new_addr;
        if (null !== last_error_code) {
            util.error('[ub.uku.js] on ' + last_error_code + ' error, changed server to ' + new_addr);
        }
        reset_count = 0;
        timeout_count = 0;
        in_changing_server = false;
    });
}
    
if (cluster.isMaster) {
    var num_CPUs = require('os').cpus().length;
    // num_CPUs = 1;

    var i;
    for (i = 0; i < num_CPUs; i++) {
        cluster.fork();
        // one note here
        // the fork() in nodejs is not as the fork() in C
        // here the fork() will run the whole code from beginning
        // not from where it is invoked
    }

    cluster.on('listening', function(worker, addr_port) {
        // use ub.uku.js as keyword for searching in log files
        util.log('[ub.uku.js] Worker ' + worker.process.pid + ' is now connected to ' + addr_port.address + ':' + addr_port.port);
    });

    cluster.on('exit', function(worker, code, signal) {
        if (signal) {
            util.log('[ub.uku.js] Worker ' + worker.process.pid + ' was killed by signal: ' + signal);
        } else if (code !== 0) {
            util.error('[ub.uku.js] Worker ' + worker.process.pid + ' exited with error code: ' + code);
            // respawn a worker process when one dies
            cluster.fork();
        } else {
            // util.error('[ub.uku.js] Worker ' + worker.process.pid + ' exited with no error; this should never happen');
            util.error('[ub.uku.js] Worker ' + worker.process.pid + ' exited.');
        }
    });

    if (!argv.production) {
        var srv = 'http://' + proxy_addr + '/proxy.pac\n';
        var msg = 'The local proxy server is running...\nPlease use this PAC file: ' + srv.underline;
        console.log(msg.green);
    } else {
        console.log('Starting in production mode...'.yellow);
    }

} else if (cluster.isWorker) {
    sogou_server_addr = sogou.new_sogou_proxy_addr();
    // console.log('default server: ' + sogou_server_addr);
    change_sogou_server();
    var change_server_timer = setInterval(change_sogou_server, 10 * 60 * 1000);  // every 10 mins
    if ('function' === typeof change_server_timer.unref) {
        change_server_timer.unref();  // doesn't exist in nodejs v0.8
    }

    http.createServer(function(client_request, client_response) {
        client_request.on('error', function(err) {
            util.error('[ub.uku.js] client_request error: (' + err.code + ') ' + err.message, err.stack);
        });
        client_response.on('error', function(err) {  // does this work?
            util.error('[ub.uku.js] client_response error: (' + err.code + ') ' + err.message, err.stack);
        });

        if (!argv.production) {
            console.log('[ub.uku.js] ' + client_request.connection.remoteAddress + ': ' + client_request.method + ' ' + client_request.url.underline);
        }

        if (!shared_tools.string_starts_with(client_request.url, '/proxy') &&
                !shared_tools.string_starts_with(client_request.url, 'http')) {
            if (client_request.url === '/favicon.ico') {
                client_response.writeHead(404, {
                    'Cache-Control': 'public, max-age=2592000'
                });
                client_response.end();
                return;
            }

            if (client_request.url === '/crossdomain.xml') {
                client_response.writeHead(200, {
                    'Content-Type': 'text/xml',
                    'Cache-Control': 'public, max-age=2592000'
                });
                client_response.end('<?xml version="1.0" encoding="UTF-8"?>\r\n' +
                        '<cross-domain-policy><allow-access-from domain="*"/></cross-domain-policy>');
                return;
            }

            if (client_request.url === '/robots.txt') {
                client_response.writeHead(200, {
                    'Content-Type': 'text/plain',
                    'Cache-Control': 'public, max-age=2592000'
                });
                client_response.end('User-agent: *\r\nDisallow: /\r\n');
                return;
            }

            if (client_request.url === '/status') {
                client_response.writeHead(200, {
                    'Content-Type': 'text/plain',
                    'Cache-Control': 'private, max-age=0, must-revalidate'
                });
                if (argv.production) {
                    client_response.end('OK\r\nProduction');
                } else {
                    client_response.end('OK');
                }
                return;
            }

            client_response.writeHead(403, {
                'Cache-Control': 'public, max-age=14400'
            });
            client_response.end();
            return;<commit_msg>add 1hr cache to the status page to lower server load<commit_after>#!/usr/bin/env node

/*
 * Allow you smoothly surf on many websites blocking non-mainland visitors.
 * Copyright (C) 2012, 2013 <NAME> http://zhuzhu.org
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */


var util = require('util');
var http = require('http');
http.globalAgent.maxSockets = Infinity;
var cluster = require('cluster');

var colors = require('colors');
var argv = require('optimist')
    .default('port', 8888)
    .boolean('local_only')
    .boolean('production')
    .argv
;

var sogou = require('../shared/sogou');
var shared_tools = require('../shared/tools');
var server_utils = require('./utils');


var local_addr, local_port, proxy_addr;
if (!argv.production) {
    local_port = argv.port;
    if (argv.local_only) {
        local_addr = '127.0.0.1';
        proxy_addr = '127.0.0.1:' + local_port;
    } else {
        local_addr = '0.0.0.0';
        proxy_addr = server_utils.get_first_external_ip() + ':' + local_port;
    }
} else {
    local_port = process.env.PORT || 8888;
    local_addr = '0.0.0.0';
    proxy_addr = 'proxy.uku.im:80';
}
var pac_file_content = shared_tools.url2pac(require('../shared/urls').url_list, proxy_addr);


// what are the life cycles of variables in nodejs?
var sogou_server_addr;
var reset_count = 0, MAX_RESET_COUNT = 1;
var timeout_count = 0, MAX_TIMEOUT_COUNT = 4;
var in_changing_server = false, last_error_code = null;
function change_sogou_server(error_code) {
    if (true === in_changing_server) {
        return;  // should already be in the process of changing new server
    }
    in_changing_server = true;

    if ('string' === typeof error_code) {
        last_error_code = error_code;
    } else {
        last_error_code = null;
    }
    server_utils.renew_sogou_server(function(new_addr) {
        sogou_server_addr = new_addr;
        if (null !== last_error_code) {
            util.error('[ub.uku.js] on ' + last_error_code + ' error, changed server to ' + new_addr);
        }
        reset_count = 0;
        timeout_count = 0;
        in_changing_server = false;
    });
}
    
if (cluster.isMaster) {
    var num_CPUs = require('os').cpus().length;
    // num_CPUs = 1;

    var i;
    for (i = 0; i < num_CPUs; i++) {
        cluster.fork();
        // one note here
        // the fork() in nodejs is not as the fork() in C
        // here the fork() will run the whole code from beginning
        // not from where it is invoked
    }

    cluster.on('listening', function(worker, addr_port) {
        // use ub.uku.js as keyword for searching in log files
        util.log('[ub.uku.js] Worker ' + worker.process.pid + ' is now connected to ' + addr_port.address + ':' + addr_port.port);
    });

    cluster.on('exit', function(worker, code, signal) {
        if (signal) {
            util.log('[ub.uku.js] Worker ' + worker.process.pid + ' was killed by signal: ' + signal);
        } else if (code !== 0) {
            util.error('[ub.uku.js] Worker ' + worker.process.pid + ' exited with error code: ' + code);
            // respawn a worker process when one dies
            cluster.fork();
        } else {
            // util.error('[ub.uku.js] Worker ' + worker.process.pid + ' exited with no error; this should never happen');
            util.error('[ub.uku.js] Worker ' + worker.process.pid + ' exited.');
        }
    });

    if (!argv.production) {
        var srv = 'http://' + proxy_addr + '/proxy.pac\n';
        var msg = 'The local proxy server is running...\nPlease use this PAC file: ' + srv.underline;
        console.log(msg.green);
    } else {
        console.log('Starting in production mode...'.yellow);
    }

} else if (cluster.isWorker) {
    sogou_server_addr = sogou.new_sogou_proxy_addr();
    // console.log('default server: ' + sogou_server_addr);
    change_sogou_server();
    var change_server_timer = setInterval(change_sogou_server, 10 * 60 * 1000);  // every 10 mins
    if ('function' === typeof change_server_timer.unref) {
        change_server_timer.unref();  // doesn't exist in nodejs v0.8
    }

    http.createServer(function(client_request, client_response) {
        client_request.on('error', function(err) {
            util.error('[ub.uku.js] client_request error: (' + err.code + ') ' + err.message, err.stack);
        });
        client_response.on('error', function(err) {  // does this work?
            util.error('[ub.uku.js] client_response error: (' + err.code + ') ' + err.message, err.stack);
        });

        if (!argv.production) {
            console.log('[ub.uku.js] ' + client_request.connection.remoteAddress + ': ' + client_request.method + ' ' + client_request.url.underline);
        }

        if (!shared_tools.string_starts_with(client_request.url, '/proxy') &&
                !shared_tools.string_starts_with(client_request.url, 'http')) {
            if (client_request.url === '/favicon.ico') {
                client_response.writeHead(404, {
                    'Cache-Control': 'public, max-age=2592000'
                });
                client_response.end();
                return;
            }

            if (client_request.url === '/crossdomain.xml') {
                client_response.writeHead(200, {
                    'Content-Type': 'text/xml',
                    'Cache-Control': 'public, max-age=2592000'
                });
                client_response.end('<?xml version="1.0" encoding="UTF-8"?>\r\n' +
                        '<cross-domain-policy><allow-access-from domain="*"/></cross-domain-policy>');
                return;
            }

            if (client_request.url === '/robots.txt') {
                client_response.writeHead(200, {
                    'Content-Type': 'text/plain',
                    'Cache-Control': 'public, max-age=2592000'
                });
                client_response.end('User-agent: *\r\nDisallow: /\r\n');
                return;
            }

            if (client_request.url === '/status') {
                client_response.writeHead(200, {
                    'Content-Type': 'text/plain',
                    'Cache-Control': 'public, max-age=3600'
                    // 'Cache-Control': 'private, max-age=0, must-revalidate'
                });
                if (argv.production) {
                    client_response.end('OK\r\nProduction');
                } else {
                    client_response.end('OK');
                }
                return;
            }

            client_response.writeHead(403, {
                'Cache-Control': 'public, max-age=14400'
            });
            client_response.end();
            return;
{'fraction_non_alphanumeric': 0.1353440541100145, 'fraction_numerical': 0.01497687901166402, 'mean_word_length': 2.1741511500547643, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 2, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2592271', 'n_tokens_mistral': 4562, 'n_tokens_neox': 4296, 'n_words': 1050}