<commit_before>
/*   acreboot.js - implementation of acre host environment using liveconnect and java */

/**
 * The acre toplevel namespace.
 * This holds the Javascript functions used by acre scripts
 *
 * @name acre
 * @namespace
 */

if (typeof acre == 'undefined') {
    acre = {};
}

(function () {

var startup_time = new Date();

// make a private copy of the objects before
//  they are wiped out from the global namespace.
// accessible reference to it.

var _topscope = this;
var _hostenv = PROTECTED_HOSTENV;
var _request = ACRE_REQUEST;
var _domparser = new DOMParser();
var _ks = new KeyStore();
var _file = File;
var _json = new JSON();
var _cache = new Cache();

// XXX there should be a java function to sanitize the global scope
// with a whitelist instead of this blacklist.  most of the
// global properties aren't enumerable so it's impossible to
// whitelist them here within javascript

delete PROTECTED_HOSTENV;
delete ACRE_REQUEST;
delete KeyStore;
delete Cache;
delete File;
delete JSON;

delete _topscope.Packages;
delete _topscope.java;
delete _topscope.netscape;

// delete E4X entrypoint if present
if (typeof _topscope.XML != 'undefined') {
    delete _topscope.XML;
}

// obtain datastore if present and remove from scope
if (typeof DataStore != 'undefined') {
    var _datastore = new DataStore();
    delete DataStore;
}

// obtain taskqueue if present and remove from scope
if (typeof TaskQueue != 'undefined') {
    var _taskqueue = new TaskQueue();
    delete TaskQueue;
}

// obtain mailer if present and remove from scope
if (typeof MailService != 'undefined') {
    var _mailer = new MailService();
    delete MailService;
}


//----------------------------- globals ---------------------------------------

// set these aside to avoid deprecation warnings when used later
var request_url = _request.request_url;
var freebase_site_host = _request.freebase_site_host;

var _DELIMITER_HOST = _hostenv.ACRE_HOST_DELIMITER_HOST;
var _DELIMITER_PATH = _hostenv.ACRE_HOST_DELIMITER_PATH;

var _DEFAULT_HOSTS_PATH = '/freebase/apps/hosts';
var _DEFAULT_ACRE_HOST_PATH = "/z/acre";

var _DEFAULTS_HOST = _hostenv.DEFAULT_HOST_PATH.substr(2);
var _METADATA_FILE = "METADATA";
var _DEFAULT_FILE = "index";
var _DEFAULT_APP = "main." + _DELIMITER_PATH;


//------------------------------- Utils ---------------------------------------

/*
 * A subset of jQuery utils loaded as 'u':
 *   u.each, u.extend, u.isArray
 * 
 * Plus a few more:
 *   u.escape_re, u.parseUri
 */
var util_scope = {};
_hostenv.load_system_script("util.js", util_scope);
var u = util_scope.exports;

// helpers for common activities in acreboot
function make_scope(start, style) {
    // XXX find a more elegant way to do this

    var copier = object;

    if (style =='deep') {
        copier = arguments.callee;
    }

    start = start || _topscope;

    function object(o) {
        function F() {};
        F.prototype = o;
        return new F();
    }

    var scope = object(start);
    for (var a in start) {
        var o = start[a];
        if (o && o instanceof Object && !(o instanceof Function)) {
            scope[a] = copier(o);
        }
    }

    return scope;
}

function compose_req_path(host, path, query_string) {
    var req_path = '//' + host;

    if (path) {
        req_path += '/' + path;
    }

    if (query_string) {
        req_path += '?' + query_string;
    }
    
    return req_path;
};

function decompose_req_path(req_path) {
    var path_re = /^([^\/]*:)?(?:\/\/([^\/\?]*))?(?:\/?([^\?]*))?(?:\?(.*))?$/;
    var [orig_req_path, protocol, host, path, query_string] = req_path.match(path_re);
    
    if (!host) throw new Error("Path: " + orig_req_path + " is not fully-qualified");
    
    // if it's a full URL we need to turn host into new require-style host
    if (protocol) {
        // remove port
        host = host.replace(/\:\d*$/,"");
        
        // normalize host relative to current acre host
        var acre_host_re = new RegExp("^((.*)\.)?" + u.escape_re(_request.server_host_base) + "$");
        var foreign_host_re = new RegExp("^(.*\.)" + _DELIMITER_HOST + "$");
        
        var m = host.match(acre_host_re);
        if (m) {
            if (m[2]) {
                var h = m[2].match(foreign_host_re);
                if (h) {
                    // foreign host:
                    //  e.g., http://trunk.svn.dev.acre-fmdb.googlecode.com.host.acre.z --> //trunk.svn.dev.acre-fmdb.googlecode.com.
                    host = h[1];
                } else {
                    // registered short host:
                    //  e.g., http://fmdb.acre.z --> //fmdb
                    host = m[2];
                }
            } else {
                // host is ACRE_HOST_BASE:
                //  e.g., http://acre.z
                host = null;
            }
        } else {
            // registered absolute host:
            //  e.g., http://tippify.com --> //tippify.com.
            host = host + ".";
        }
    }
    
    return [host, path, query_string];
}

function file_in_path(filename, path) {
    var file_re = new RegExp(u.escape_re(filename) + "(\/.*)?$");
    var in_path = false;
    var path_info = "";
    
    if (file_re.test(path)) {
        in_path = true;
        path_info = path.replace(new RegExp("^" + u.escape_re(filename)),"");
        // XXX - add leading slash for backward-compatibility
        if (path_info === "") path_info = "/";
    } else {
        // XXX - add leading slash for backward-compatibility
        path_info = "/" + path;
    }
    
    // XXX - strip trailing slash for backward-compatibility
    if (path_info.length > 1 && path_info.substr(-1) === "/") {
        path_info = path_info.substr(0, path_info.length-1);
    }
    
    return [path_info, in_path];
}

function get_extension_metadata(name, extensions) {
    // match from longest to shortest (i.e., .mf.css before .css)
    var exts = name.split(".");
    exts.shift();
    while (exts.length) {
      var ext = exts.join(".");
      var ext_data = {};
      if (ext && extensions && extensions[ext]) {
          ext_data = extensions[ext];
          break;
      }
      exts.shift();
    }
    
    return ext_data;
};


// helpers for backward-compatibility with the days of script_ids
// it sure would be nice to get rid of these...
function namespace_to_host(namespace) {
    var host_re = new RegExp(u.escape_re('^' + _DEFAULT_HOSTS_PATH));
    var acre_host_re = new RegExp(u.escape_re('^' + _DEFAULT_ACRE_HOST_PATH));
    
    if (host_re.test(namespace)) {
        namespace = namespace.replace(host_re, "");
        if (acre_host_re.test(namespace)) {
            namespace = namespace.replace(acre_host_re, "").replace(/^\//, "");
        }
    } else {
        namespace = _DELIMITER_PATH + namespace;
    }
    return namespace.split("/").reverse().join(".");
}

function host_to_namespace(host) {
    var path = null;
    
    var host_parts = host.split(".");
    var trailing_host_part = host_parts.pop();
    if (trailing_host_part === _DELIMITER_PATH) {
        path = "/" + host_parts.reverse().join("/");
    } else if (trailing_host_part === '') {
        path = _DEFAULT_HOSTS_PATH + "/" + host_parts.reverse().join("/");
    } else {
        host_parts.push(trailing_host_part);
        path = _DEFAULT_HOSTS_PATH + _DEFAULT_ACRE_HOST_PATH + "/" + host_parts.reverse().join("/");
    }
    
    return path;
}

function split_script_id(script_id) {
    var namespace = script_id.split('/');
    var f = namespace.pop();
    namespace = namespace.join('/');
    return [namespace, f];
}

function req_path_to_script_id(req_path) {
    var [host, script] = decompose_req_path(req_path);
    script = script.split("/").pop();
    return host_to_namespace(host) + "/" + script;
}


// random number generator used in CSRF protection
function get_random() {
    return Math.floor(Math.random()*10e15).toString();
};

function get_csrf_secret(that) {
    if (!_request.csrf_secret) {
        var key = _ks.get_key("csrf", _request.app_project);
        if (key && key[1]) {
            _request.csrf_secret = key[1];
        } else {
            _request.csrf_secret = get_random();
            _ks.put_key("csrf", _request.app_project, null, _request.csrf_secret);
        }
    }
    return _request.csrf_secret;
};


//--------------------------------- syslog --------------------------------------------

var _syslog = function(level, mesg, event_name) {
    if (typeof event_name == 'undefined') event_name = null;
    if (typeof mesg == 'undefined' || mesg === null) mesg = "undefined";
    _hostenv.syslog(level, event_name, mesg);
};

var syslog   = function (mesg, event_name) { return _syslog("INFO",mesg,event_name); };
syslog.debug = function (mesg, event_name) { return _syslog("DEBUG",mesg,event_name); };
syslog.info  = function (mesg, event_name) { return _syslog("INFO",mesg,event_name); };
syslog.warn  = function (mesg, event_name) { return _syslog("WARN",mesg,event_name); };
syslog.error = function (mesg, event_name) { return _syslog("ERROR",mesg,event_name); };

acre.syslog = syslog;

//--------------------------------- dev mode ------------------------------------------

/**
 *  this should be enabled for development only!
 */
var _dev = {};

_dev.test_internal = function (ename) {
    return _hostenv.dev_test_internal(ename);
};
_dev.syslog = syslog;

// developer entrypoints are invisible by default
if (_hostenv.ACRE_DEVELOPER_MODE) {
    acre._dev = _dev;
}

// ------------------------------- acre ----------------------------------------

acre.version = new String(_request.version);

/**
 *   begin the http response.
 *
 *   The Content-type header, if specified, is handled specially.
 *   If it is any text/* type, the charset specified in the header will
 *   be used to encode the response.  If no charset= parameter is given,
 *   the response will be encoded as UTF-8, and "; charset=utf-8" is added
 *   to the Content-type header before sending the response.
 *   <br/>
 *   For content types other than text/* the text is always encoded as UTF-8.
 *
 *   @param status is the response status (e.g. 200)
 *   @param headers is a javascript object
 */
acre.start_response = function (status, headers) {

    if (typeof status == 'undefined') {
        status = 200;
    } else if (typeof status == 'string') {
        status = parseInt(status, 10);
    } else if (typeof status == 'number') {
        acre.response.status = status;
    } else {
        throw new Error('acre.start_response: status "' + status.toSource() + '" must be an integer or string');
    }

    if (typeof headers == 'object') {
        // stupid merge - arguments override existing headers
        // some headers are patched in later though...
        u.extend(acre.response.headers, headers);
    }
};

/**
 *   add output to the http response.  multiple arguments will be appended.
 *   @param str string or markup to append
 */
acre.write = function () {
    u.each(arguments, function(i, arg) {
        // XXX - shouldn't have handler-specific code here
        if (typeof arg == 'object' && arg !== null &&
            (typeof arg.toMarkupList === 'function' || typeof arg.toMarkup === 'function')) {
            arg = acre.markup.stringify(arg);
        }

        if (typeof arg != 'string' && !(arg instanceof Binary))
            arg = '' + arg;

        _hostenv.write(arg);        
    });
};

/**
 * exit the current acre request.<br/><br/>
 *
 * this works by throwing an exception,
 * that exception propagates across acre.load* calls.<br/><br/>
 *
 * Currently, user code can catch that exception.
 * we may wish to change this behavior since most people expect
 * to catch errors rather than normal termination.<br/><br/>
 *
 * if an acre script exits before calling <b>acre.start_response()</b>,
 * an error response is generated.
 */
acre.exit = function () {
    throw new _hostenv.AcreExitException();
};

acre.wait = function(millis) {
    _hostenv.do_wait(millis);
}

// ------------------------------- acre.host -----------------------------------------

if (_request.server.indexOf("jetty") > -1) {
    var server_type = "acre_server";
} else if (_request.server.indexOf("app engine") > -1) {
    var server_type = "appengine";
} else {
    var server_type = new String(_request.server);
}

acre.host = {
    protocol : _request.server_protocol,
    name : _request.server_host_base,
    dev_name : _request.server_host,
    port : _request.server_port,
    server : server_type
};


//-------------------------------- acre.request -----------------------------------------

acre.request = {
    server_name : _request.request_server_name,
    server_port : _request.request_server_port,
    url : _request.request_url,
    base_url : _request.request_base_url,
    app_url : _request.request_app_url,
    protocol : _request.server_protocol, // XXX: how do we get the request protocol?
    method : _request.request_method,
    headers : _request.headers,
    base_path : _request.request_path_info.replace(new RegExp(u.escape_re(_request.path_info)+"$"), ""),    

    // these can be re-set by acre.route
    body : _request.request_body,
    path_info : _request.path_info,
    query_string : _request.query_string,
    
    // set in request params
    params: {},
    body_params: {},
    
    start_time : _request.request_start_time,
    cookies : {},
    skip_cache : false
};

// if we get 'cache-control: no-cache' in the request headers (i.e., shift-reload),
// then set a flag that appfetchers can key off of.
// some browsers will by default issue this header for *all* XHRs, however,
// which can make requests very slow. the work-around is that if an 
// x-acre-cache-control header is present, then we do not set the flag
if ('cache-control' in acre.request.headers && !('x-acre-cache-control' in acre.request.headers)) {
    if (/no-cache/.test(acre.request.headers['cache-control'])) {
        acre.request.skip_cache = true;
    }
}

// Deal with old metaweb cookie weirdness (TODO - destroy this asap)
// Close to avoid leaking variables into the scope
_request.mwlt_mode = false;
(function () {
    var _MWAUTH_HOSTS = _hostenv.ACRE_ALLOW_MWAUTH_HOST_SUFFIX.split(' ');
     var ok = false;
     var host = acre.request.server_name;
     for (var a in _MWAUTH_HOSTS) {
         var h = _MWAUTH_HOSTS[a];
         if (host.match(h+'$')){ // && !(host.match('\.acrenet\.metaweb\.com$'))) {
             ok = true;
             break;
         }
     }
     if (ok)
         _request.mwlt_mode = true;

    for (var name in _request.cookies) {
        if (name == 'metaweb-user-info') {
            var user_cookie = _request.cookies['metaweb-user-info'].value;

            function get_part(key) {
                var cookie_part = new RegExp('\\|('+key+')_(.*?)\\|');
                var match = cookie_part.exec(user_cookie);
                return decodeURIComponent(match[2]);
            }

            acre.request.user_info = {
               guid : get_part('g'),
               id : get_part('p'),
               name : get_part('u')
            };
        } else if (name == 'metaweb-user') {
            if (!ok) {
                syslog.debug("deleting metaweb-user cookie");
                delete _request.cookies['metaweb-user'];
            } else {
                acre.request.cookies[name] = _request.cookies[name].value;
            }
        } else {
            acre.request.cookies[name] = _request.cookies[name].value;
        }
    }

})();

/**
 * reset acre.request.query_string & body
 * 
 * Also parse params into objects for easy look-up:
 *   query_string --> acre.request.params
 *   body --> acre.request.body_params
 */
function set_request_params() {
    var query_string = acre.request.query_string;
    var body = acre.request.body;
    
    try {
        acre.request.params = (typeof query_string == 'string') ? acre.form.decode(query_string) : {};
        acre.request.body_params = (typeof body == 'string' && typeof acre.request.headers['content-type'] == 'string' && acre.request.headers['content-type'].match(/^application\/x-www-form-urlencoded/)) ? acre.form.decode(body) : {};
    } catch (e) {
        acre.request.params = {};
        acre.request.body_params = {};
        console.warn("Invalid request: parameters were not properly encoded");
    }
}


// ------------------------------- acre.respose -----------------------------------------

function AcreResponse() {
    this.status = 200;
    this.headers = {};
    this.cookies = {};
}

// Variables set by internal private helpers
var AcreResponse_availability = false;
var AcreResponse_max_age = 0;
var AcreResponse_vary_cookies = {};
var AcreResponse_session = acre.request.cookies['ACRE_SESSION'];
var AcreResponse_set_session_cookie = false;

var AcreResponse_validate_header = function(name, value) {
    // according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2
    var http_token_re = /[-a-zA-Z0-9!#$%&'*+.^_`|~]+/; // ']

    // check that name and value are valid
    if (!http_token_re.test(name)) {
        throw new Error('illegal HTTP header name');
    }

    name = name.toLowerCase();

    if (/[\r\n]/.test(value)) {
        throw new Error('newline is illegal in HTTP header value');
    }
};

// AcreResponse methods are exposed to user scripts as acre.response.*
AcreResponse.prototype.set_header = function (name, value) {
    name = name.toLowerCase();
    AcreResponse_validate_header(name, value);
    this.headers[name] = value;
};

AcreResponse.prototype.add_header = function (name, value) {
    name = name.toLowerCase();
    AcreResponse_validate_header(name, value);

    // RFC2616 sec 4.2: It MUST be possible to combine the multiple
    //   header fields into one "field-name: field-value" pair, without
    //   changing the semantics of the message, by appending each
    //   subsequent field-value to the first, each separated by a comma.
    if (!(name in this.headers)) {
        this.headers[name] = value;
    } else if (u.isArray(this.headers[name])) {
        this.headers[name].push(value);
    } else {
        this.headers[name] = [this.headers[name], value];
    }
};

AcreResponse.prototype.set_header_default = function (name, value) {
    name = name.toLowerCase();
    if (name in this.headers) {
        return;
    }
    this.set_header(name, value);
};

AcreResponse.prototype.clear_cookie = function (name, options) {
    if (typeof options == 'undefined') {
        options = {};
    }
    var clear_cookie_options = { domain:1, path:1, secure:1 };
    var newopts = {};
    for (k in clear_cookie_options) {
        if (k in options) {
            newopts[k] = options[k];
        }
    }
    newopts.max_age = 0;
    AcreResponse_vary_cookies[name] = 1;
    this.set_cookie(name, '', newopts);
};

// options is an object with valid properties being  domain, path, secure, max_age, expires
// max_age is preferred to expires, it gives a lifetime in seconds
// max_age=0 causes the cookie to be cleared
// expires is accepted as a string or a javascript Date object
// max_age causes an expires attribute to be generated too for IE's sake.
AcreResponse.prototype.set_cookie = function (name, value, options) {
    var valid_cookie_options = { domain:1, path:1, secure:1, max_age:1, expires:1 };

    if (typeof options == 'undefined') {
        options = {};
    }

    if (typeof options.max_age != 'undefined' && typeof options.max_age != 'number') {
        throw new Error('set_cookie() max_age must be a number');
    }

    // TODO should check value for non-cookie ok strings
    var cookie = { name: name, value: value };

    for (var k in options) {
        if (!(k in valid_cookie_options)) {
            throw new Error('Invalid cookie option "' + k + '"');
        }
        cookie[k] = options[k];
    }

    // the acre java code understands max-age= but not expires=
    // the jetty cookie manager converts max-age to expires in the response.
    if (typeof cookie.max_age == 'undefined') {
        // calculate max-age= from expires= if present
        if (typeof cookie.expires != 'undefined') {
            var d = cookie.expires;
            if (!(d instanceof Date)) {
                d = Date.parse(d);
            }

            // find the time from now until the expires= time
            var delta = d.getTime() - (new Date()).getTime();
            if (delta <= 0) {
                cookie.max_age = 0;
            } else {
                cookie.max_age = Math.ceil(delta / 1000.0);
            }
        }
    } else if (typeof cookie.expires != 'undefined') {
        // complain if both max-age= and expires= are given
        console.warn('Cookie "max_age" attribute overides "expires" attribute');
    }

    AcreResponse_vary_cookies[name] = 1;
    this.cookies[name] = cookie;
};

AcreResponse.prototype.set_cache_policy = function (policy) {
    switch (policy) {
        case 'fresh':
            AcreResponse_max_age = 0;
            AcreResponse_availability = false;
            break;
        case 'fast':
            AcreResponse_max_age = 3600;
            AcreResponse_availability = true;
            break;
    }
};

AcreResponse.prototype.set_policy =
    AcreResponse.prototype.set_cache_policy; // deprecated

// these AcreResponse_* functions are called during _hostenv.finish_response()
var AcreResponse_generate_date = function (d) {
    function padz(n) {
        return n > 10 ? n : '0' + n;
    }

    var MONTH_NAMES = [
        "January", "February", "March",
        "April", "May", "June",
        "July", "August", "September",
        "October", "November", "December"
    ];

    var WEEKDAY_NAMES = [
        "Sunday", "Monday", "Tuesday",
        "Wednesday", "Thursday", "Friday",
        "Saturday"
    ];

    var dayname = WEEKDAY_NAMES[d.getUTCDay()].slice(0,3);
    var day = padz(d.getUTCDate());
    var monthname = MONTH_NAMES[d.getUTCMonth()].slice(0,3);
    var year = d.getUTCFullYear();
    var hour = padz(d.getUTCHours());
    var minutes = padz(d.getUTCMinutes());
    var seconds = padz(d.getUTCSeconds());

    return dayname+', '+day+' '+monthname+' '+year+' '+hour+':'+ minutes +':'+ seconds+' UTC';
};

var AcreResponse_set_expires = function (that) {
    var expires = new Date(acre.request.start_time.getTime()+(AcreResponse_max_age*1000));
    var d = AcreResponse_generate_date(expires);

    that.set_header_default('Expires', d);
};

var AcreResponse_set_last_modified = function (that) {
    var d = AcreResponse_generate_date(acre.request.start_time);
    that.set_header_default('Last-Modified', d);
};

var AcreResponse_set_cache_control = function (that) {
    var cc = '';

    if (AcreResponse_availability == true) {
        cc += 'public';
        cc += ', no-cache="Set-Cookie"';

        cc += ', max-age='+AcreResponse_max_age;
        for (var a in AcreResponse_vary_cookies) {
            cc += ', maxage-vary-cookie="'+AcreResponse_max_age+'|'+a+'"';
        }

    } else {
        cc += 'private, no-cache, max-age=0';
    }

    that.set_header_default('cache-control', cc);
};

var AcreResponse_set_vary = function (that) {
    if (acre.response.headers.vary){
        var vary = acre.response.headers.vary.split(", ");
    } else {
        var vary = [];
    }
    
    for (key in AcreResponse_vary_cookies) {
        vary.push('Cookie');
        break;
    }

    if (vary.length > 0) {
        acre.response.set_header('Vary', vary.join(', '));
    }
};

var AcreResponse_set_metaweb_vary = function (that) {
    var vary = [];
    for (var c in AcreResponse_vary_cookies.__count__) {
        vary.push('Cookie['+c+']');
    }

    if (vary.length > 0) {
        acre.response.set_header_default('x-metaweb-vary', vary.join(', '));
    }
};

var AcreResponse_get_session = function(that) {
    if (typeof AcreResponse_session === 'undefined') {
        AcreResponse_session = get_random();
        AcreResponse_set_session_cookie = true;
    }
    return AcreResponse_session;
};

var AcreResponse_set_session = function(that) {
    if (AcreResponse_set_session_cookie) {
        that.set_cookie("ACRE_SESSION", AcreResponse_session);
    }
};

acre.response = null;


// --------------------------- acre.errors ---------------------------------

/*
 * Setup the acre.error object in case the request is for an error page
 */
if (_request.error_info) {
    acre.error = _request.error_info;

    // error-specific info is passed as a JSON string - decode it here
    var info_json = acre.error.info_json;
    acre.error.info = _json.parse('['+acre.error.info_json+']')[0];
    delete acre.error.info_json;
}

acre.errors = {};

/**
 *  exception class to allow bailing out early, e.g.
 *   after responding quickly with an error code.
 *  this api is likely to change so this exception
 *  should be thrown by calling acre.exit(), not
 *  by thowing it directly.  it is documented here
 *  because developers might notice this exception
 *  getting caught in their javascript "catch" blocks.
 *  if this occurs, the best thing is probably to
 *  re-throw the exception.
 */
acre.errors.AcreRouteException = function () {};
acre.errors.AcreRouteException.prototype = new Error('acre.route');

acre.errors.AcreExitException = function () {};
acre.errors.AcreExitException.prototype = new Error('acre.exit');
_hostenv.AcreExitException = acre.errors.AcreExitException;

acre.errors.URLError = function (message) {
    this.message = message;
};
acre.errors.URLError.prototype = new Error('acre.errors.URLError');
acre.errors.URLError.prototype.name = 'acre.errors.URLError';
_hostenv.URLError = acre.errors.URLError;

acre.errors.URLTimeoutError = function (message) {
    this.message = message;
};
acre.errors.URLTimeoutError.prototype = new acre.errors.URLError('acre.errors.URLTimeoutError');
acre.errors.URLTimeoutError.prototype.name = 'acre.errors.URLTimeoutError';
_hostenv.URLTimeoutError = acre.errors.URLTimeoutError;

_topscope.AcreExitException = acre.errors.AcreExitException; // deprecated
_topscope.URLError = acre.errors.URLError; // deprecated

_hostenv.Error = Error;


// ------------------------------- acre.environ -----------------------------------

// deprecated
function set_environ() {
    // avoid deprecation warnings
    // if already set up (acre.route)
    if (acre.environ) {
        delete acre.environ.query_string;
        delete acre.environ.params;
        delete acre.environ.body_params;
        delete acre.environ.path_info;
        delete acre.environ.script_name;
        delete acre.environ.script_id;
        delete acre.environ.script_namespace;
        delete acre.environ.user_info;
        delete acre.environ;
    }
    
    acre.environ = {
        version: _request.version,
        script_name: _request.script_name,
        script_id: _request.script_id,
        script_namespace: _request.script_namespace,
        path_info: _request.path_info,
        query_string: _request.query_string,
        headers: _request.headers,
        cookies: _request.cookies,
        server_name: _request.server_name,
        server_host: _request.server_host,
        server_host_base: _request.server_host_base,
        server_port: _request.server_port,
        server_protocol: _request.server_protocol,
        request_body: _request.request_body,
        request_method: _request.request_method,
        request_path_info: _request.request_path_info,
        request_server_name: _request.request_server_name,
        request_server_port: _request.request_server_port,
        request_start_time: _request.request_start_time,
        request_url: _request.request_url,
        request_app_url: _request.request_app_url,
        request_base_url: _request.request_base_url,
        freebase_service_url: _request.freebase_service_url,
        freebase_site_host: _request.freebase_site_host
    };
    acre.environ.params = acre.request.params;
    acre.environ.body_params = acre.request.body_params;
    
    if (acre.request.user_info) {
        acre.environ.user_info = acre.request.user_info;
    }
};


// ------------------------------------ JSON ------------------------------

_topscope.JSON = {
    stringify: function(obj, resolver, space) {
        try {
            return _json.stringify(obj, resolver, space);
        } catch (e) {
          var message = e.message.indexOf("JSON.stringify:") == 0
                          ? e.message
                          : e.message.replace(/^[^:]*:\s*/, '');
            throw new Error(message);
        }
    },
    parse: function(str, reviver) {
        try {
            if (typeof reviver !== "undefined" && reviver != null) {
                console.warn("Acre currently ignores 'reviver' in JSON.parse()");
            }

            return _json.parse(str);
        } catch (e) {
            // Everything after the first :, if present
            throw new Error(e.message.replace(/^[^:]*:\s*/, ''));
        }
    }
};


// ------------------------------ xml/html parsers ----------------------------

if (typeof acre.xml == 'undefined') {
    acre.xml = {};
}

if (typeof acre.html == 'undefined') {
    acre.html = {};
}

/**
 *  Parse a string containing XML into a DOM Document object. <br/><br/>
 *
 *  The returned Document object complies to the W3C EcmaScript bindings for
 *  the DOM.
 *
 *  @param xml_str the XML string to be parsed
 */
acre.xml.parse = function(xml_str) {
    return _domparser.parse_string(xml_str, "xml");
};

/**
 *  Parse a string containing XML into a DOM Document object and support namespaces. <br/><br/>
 *
 *  The returned Document object complies to the W3C EcmaScript bindings for
 *  the DOM.
 *
 *  @param xml_str the XML string to be parsed
 */
acre.xml.parseNS = function(xml_str) {
    return _domparser.parse_ns_string(xml_str, "xml");
};

/**
 *  Parse a string containing HTML into a DOM Document object. <br/><br/>
 *
 *  The returned Document object complies to the W3C EcmaScript bindings for
 *  the DOM.
 *
 *  @param html_str the HTML string to be parsed
 */
acre.html.parse = function(html_str) {
    return _domparser.parse_string(html_str, "html");
};


// ------------------------ acre.hash ------------------------------

if (typeof acre.hash == 'undefined') {
    acre.hash = {};
}

acre.hash.hex_sha1 = function (s) {
    return _hostenv.hash("SHA-1", s, true);
};

acre.hash.b64_sha1 = function (s) {
    return _hostenv.hash("SHA-1", s, false);
};

acre.hash.hex_md5 = function (s) {
    return _hostenv.hash("MD5", s, true);
};

acre.hash.b64_md5 = function (s) {
    return _hostenv.hash("MD5", s, false);
};

acre.hash.hex_hmac_sha1 = function (key, data) {
    return _hostenv.hmac("HmacSHA1", key, data, true);
};

acre.hash.b64_hmac_sha1 = function (key, data) {
    return _hostenv.hmac("HmacSHA1", key, data, false);
};

acre.hash.hex_hmac_md5 = function (key, data) {
    return _hostenv.hmac("HmacMD5", key, data, true);
};

acre.hash.b64_hmac_md5 = function (key, data) {
    return _hostenv.hmac("HmacMD5", key, data, false);
};


// ------------------------------------------------------------------------

// used by the keystore and the cache to partition data based on the app_guid

_request.app_project = null;

// ------------------------ keystore --------------------------------------

if (typeof acre.keystore == 'undefined') {
    acre.keystore = {};
}

acre.keystore.get = function (name) {
    if (_request.app_project !== null) {
        return _ks.get_key(name, _request.app_project);
    } else {
        return null;
    }
};

acre.keystore.keys = function () {
    if (_request.app_project !== null) {
        return _ks.get_keys(_request.app_project);
    } else {
        return null;
    }
};

acre.keystore.remove = function (name) {
    if (_request.app_project !== null) {
        _ks.delete_key(name, _request.app_project);
    }
};

if (_request.trusted) { 
    acre.keystore.put = function (name, token, secret) {
        if (_request.app_project !== null) {
            return _ks.put_key(name, _request.app_project, token, secret);
        } else {
            return null;
        }
    };
}

//------------------------ cache --------------------------

var cache_scope = {};
cache_scope.syslog = syslog;
cache_scope.cache = _cache;
cache_scope.acreboot = _topscope;
cache_scope.request = _request;
_hostenv.load_system_script('cache.js', cache_scope);
cache_scope.augment(acre);

//------------------------ datastore --------------------------

if (_request.trusted && _datastore) { // the _datastore object won't be available in all environments so we need to check first
    var store_scope = {};
    store_scope.syslog = syslog;
    store_scope.store = _datastore;
    _hostenv.load_system_script('datastore.js', store_scope);
    store_scope.augment(acre);
}

//------------------------ taskqueue --------------------------

if (_request.trusted && _taskqueue) { // the _taskqueue object won't be available in all environments so we need to check first
    var queue_scope = {};
    queue_scope.syslog = syslog;
    queue_scope.queue = _taskqueue;
    _hostenv.load_system_script('taskqueue.js', queue_scope);
    queue_scope.augment(acre);
}

//------------------------ mailer --------------------------

if (_request.trusted && _mailer) { // the _mailer object won't be available in all environments so we need to check first
    var mailer_scope = {};
    mailer_scope.syslog = syslog;
    mailer_scope.mailer = _mailer;
    _hostenv.load_system_script('mailservice.js', mailer_scope);
    mailer_scope.augment(acre);
}

// ------------------------------ urlfetch -----------------------------------

/**
 *   allows an acre script to fetch data from an external url.<br/><br/>
 *
 *   this may not be part of the standard api, we will probably want to
 *   restrict which urls are available to prevent abuse.<br/><br/>
 *
 *   note that response.body is a javascript string, which means it
 *   has already been decoded.  there are bugs in this process,
 *   because we don't sniff for <meta http-equiv="content-type"...> here.<br/><br/>
 *
 *   <pre>
 *   returns a response object, e.g.:
 *     {
 *       status: 200,
 *       headers: {
 *          "content-length": "14",
 *          "content-type": "text/plain; charset=UTF-8",
 *       },
 *       content_type: "text/plain",
 *       body: "hocus.\npocus.\n"
 *    }
 *    </pre><br/><br/>
 *
 *  all header names are canonicalized to lowercase
 *
 */

var _system_urlfetch = function(url,method,headers,content,sign) {
    return _urlfetch(true,url,method,headers,content,sign);
};
var _system_async_urlfetch = function(url,method,headers,content,sign) {
    return _urlfetch(true,url,method,headers,content,sign, _hostenv.urlOpenAsync);
};

// acre.urlfetch is disabled in error scripts.  the reason is
//  that error scripts are given a special extension on the
//  original script's time quota.  they can't be allowed
//  to re-enter the server or they would be able to pass on
//  that time quota to another request.
// this could be relaxed to allow users to call services
//  other than acre services.
acre.async = {};
if (!acre.error) {
    acre.urlfetch = function(url,method,headers,content,sign) {
        return _urlfetch(false,url,method,headers,content,sign);
    };
    acre.async.urlfetch = function(url, method, headers, content, sign) {
        return _urlfetch(false,url,method,headers,content,sign,_hostenv.urlOpenAsync);
    };
    acre.async.wait_on_results = function (timeout) {
        _hostenv.async_wait(timeout);
    };
} else {
    acre.urlfetch = function(url,method,headers,content,sign) {
        throw new Error('acre.urlfetch() is restricted in error scripts');
    };
    acre.async.urlfetch = function(url, method, headers, content, sign) {
        throw new Error('acre.async.urlfetch() is restricted in error scripts');
    };
    acre.async.wait_on_results = function () {
        throw new Error('acre.async.wait_on_results() is restricted in error scripts');
    };
}

/*
 * Fetches an HTTP URL.
 *
 * Parameters:
 *
 *  - system: whether or not this is a fetch made by acre directly or by the user script [required]
 *  - url: the URL to fetch [required]
 *  - method: the HTTP method to use to fetch it [optional, defaults to GET]
 *  - headers: the HTTP headers to pass along with the request [optional]
 *  - content: the HTTP payload of the request [optional]
 *  - sign: whether or not sign the request (using oauth) [optional]
 *
 * NOTE: sign can be either a boolean or an object. If a boolean
 * the oauth subsystem will look for the right key based on the domain
 * name of the request being made. If it's an object it must contain the key/secret
 * pair that identifies the application against the oauth provider and
 * must be in the form:
 *
 *     {
 *       "consumerKey" : key,
 *       "consumerSecret" : secret
 *     }
 */
var _urlfetch = function (system, url, options_or_method, headers, content, sign, _urlopener) {
    _urlopener = _urlopener || _hostenv.urlOpen;

    // exception generation would be nice here
    if (typeof url != 'string' || !url.length) {
        throw new acre.errors.URLError("'url' argument (1st) to acre.urlfetch() must be a string");
    }
    
    var method;
    var response_encoding = null;
    var callback;
    var errback;
    var timeout;
    var bless;
    var no_redirect;
    
    if (options_or_method && typeof options_or_method === 'object') {
        if (typeof headers !== 'undefined' || typeof content !== 'undefined' || typeof sign !== 'undefined') {
            throw new acre.errors.URLError("'options' argument (2nd) to acre.urlfetch() must not be followed by extra arguments");
        }

        method = options_or_method.method;
        headers = options_or_method.headers;
        content = options_or_method.content;
        sign = !!options_or_method.sign;
        response_encoding = options_or_method.response_encoding;
        callback = options_or_method.callback || function (res) { };
        errback = options_or_method.errback || function (res) { throw res; };
        timeout = parseInt(options_or_method.timeout, 10) || undefined;
        bless = !!options_or_method.bless;
        no_redirect = options_or_method.no_redirect;
    } else if (options_or_method) {
        method = options_or_method;
    }

    if (typeof sign !== 'undefined' && typeof sign !== 'boolean' && typeof sign !== 'object') {
        throw new acre.errors.URLError("'sign' argument (5th) or option to acre.urlfetch() must be a boolean or an object");
    }

    if (typeof method != 'string') method = 'GET';
    if (typeof headers != 'object') headers = {};
    if (typeof content != 'string' && !(content instanceof Binary)) content = '';
    if (typeof response_encoding != 'string') response_encoding = 'utf-8';
    if (typeof no_redirect != 'boolean') no_redirect = false;
    
    // lowercase all the header names so we don't have to worry about
    // case sensitivity
    for (var hk in headers) {
        var hkl = hk.toLowerCase();
        if (hk == hkl) continue;
        var hkv = headers[hk];
        delete headers[hk];<commit_msg>for Alex...<commit_after>
/*   acreboot.js - implementation of acre host environment using liveconnect and java */

/**
 * The acre toplevel namespace.
 * This holds the Javascript functions used by acre scripts
 *
 * @name acre
 * @namespace
 */

if (typeof acre == 'undefined') {
    acre = {};
}

(function () {

var startup_time = new Date();

// make a private copy of the objects before
//  they are wiped out from the global namespace.
// accessible reference to it.

var _topscope = this;
var _hostenv = PROTECTED_HOSTENV;
var _request = ACRE_REQUEST;
var _domparser = new DOMParser();
var _ks = new KeyStore();
var _file = File;
var _json = new JSON();
var _cache = new Cache();

// XXX there should be a java function to sanitize the global scope
// with a whitelist instead of this blacklist.  most of the
// global properties aren't enumerable so it's impossible to
// whitelist them here within javascript

delete PROTECTED_HOSTENV;
delete ACRE_REQUEST;
delete KeyStore;
delete Cache;
delete File;
delete JSON;

delete _topscope.Packages;
delete _topscope.java;
delete _topscope.netscape;

// delete E4X entrypoint if present
if (typeof _topscope.XML != 'undefined') {
    delete _topscope.XML;
}

// obtain datastore if present and remove from scope
if (typeof DataStore != 'undefined') {
    var _datastore = new DataStore();
    delete DataStore;
}

// obtain taskqueue if present and remove from scope
if (typeof TaskQueue != 'undefined') {
    var _taskqueue = new TaskQueue();
    delete TaskQueue;
}

// obtain mailer if present and remove from scope
if (typeof MailService != 'undefined') {
    var _mailer = new MailService();
    delete MailService;
}


//----------------------------- globals ---------------------------------------

// set these aside to avoid deprecation warnings when used later
var request_url = _request.request_url;
var freebase_site_host = _request.freebase_site_host;

var _DELIMITER_HOST = _hostenv.ACRE_HOST_DELIMITER_HOST;
var _DELIMITER_PATH = _hostenv.ACRE_HOST_DELIMITER_PATH;

var _DEFAULT_HOSTS_PATH = '/freebase/apps/hosts';
var _DEFAULT_ACRE_HOST_PATH = "/z/acre";

var _DEFAULTS_HOST = _hostenv.DEFAULT_HOST_PATH.substr(2);
var _METADATA_FILE = "METADATA";
var _DEFAULT_FILE = "index";
var _DEFAULT_APP = "main." + _DELIMITER_PATH;


//------------------------------- Utils ---------------------------------------

/*
 * A subset of jQuery utils loaded as 'u':
 *   u.each, u.extend, u.isArray
 * 
 * Plus a few more:
 *   u.escape_re, u.parseUri
 */
var util_scope = {};
_hostenv.load_system_script("util.js", util_scope);
var u = util_scope.exports;

// helpers for common activities in acreboot
function make_scope(start, style) {
    // XXX find a more elegant way to do this

    var copier = object;

    if (style =='deep') {
        copier = arguments.callee;
    }

    start = start || _topscope;

    function object(o) {
        function F() {};
        F.prototype = o;
        return new F();
    }

    var scope = object(start);
    for (var a in start) {
        var o = start[a];
        if (o && o instanceof Object && !(o instanceof Function)) {
            scope[a] = copier(o);
        }
    }

    return scope;
}

function compose_req_path(host, path, query_string) {
    var req_path = '//' + host;

    if (path) {
        req_path += '/' + path;
    }

    if (query_string) {
        req_path += '?' + query_string;
    }
    
    return req_path;
};

function decompose_req_path(req_path) {
    var path_re = /^([^\/]*:)?(?:\/\/([^\/\?]*))?(?:\/?([^\?]*))?(?:\?(.*))?$/;
    var [orig_req_path, protocol, host, path, query_string] = req_path.match(path_re);
    
    if (!host) throw new Error("Path: " + orig_req_path + " is not fully-qualified");
    
    // if it's a full URL we need to turn host into new require-style host
    if (protocol) {
        // remove port
        host = host.replace(/\:\d*$/,"");
        
        // normalize host relative to current acre host
        var acre_host_re = new RegExp("^((.*)\.)?" + u.escape_re(_request.server_host_base) + "$");
        var foreign_host_re = new RegExp("^(.*\.)" + _DELIMITER_HOST + "$");
        
        var m = host.match(acre_host_re);
        if (m) {
            if (m[2]) {
                var h = m[2].match(foreign_host_re);
                if (h) {
                    // foreign host:
                    //  e.g., http://trunk.svn.dev.acre-fmdb.googlecode.com.host.acre.z --> //trunk.svn.dev.acre-fmdb.googlecode.com.
                    host = h[1];
                } else {
                    // registered short host:
                    //  e.g., http://fmdb.acre.z --> //fmdb
                    host = m[2];
                }
            } else {
                // host is ACRE_HOST_BASE:
                //  e.g., http://acre.z
                host = null;
            }
        } else {
            // registered absolute host:
            //  e.g., http://tippify.com --> //tippify.com.
            host = host + ".";
        }
    }
    
    return [host, path, query_string];
}

function file_in_path(filename, path) {
    var file_re = new RegExp(u.escape_re(filename) + "(\/.*)?$");
    var in_path = false;
    var path_info = "";
    
    if (file_re.test(path)) {
        in_path = true;
        path_info = path.replace(new RegExp("^" + u.escape_re(filename)),"");
        // XXX - add leading slash for backward-compatibility
        if (path_info === "") path_info = "/";
    } else {
        // XXX - add leading slash for backward-compatibility
        path_info = "/" + path;
    }
    
    // XXX - strip trailing slash for backward-compatibility
    if (path_info.length > 1 && path_info.substr(-1) === "/") {
        path_info = path_info.substr(0, path_info.length-1);
    }
    
    return [path_info, in_path];
}

function get_extension_metadata(name, extensions) {
    // match from longest to shortest (i.e., .mf.css before .css)
    var exts = name.split(".");
    exts.shift();
    while (exts.length) {
      var ext = exts.join(".");
      var ext_data = {};
      if (ext && extensions && extensions[ext]) {
          ext_data = extensions[ext];
          break;
      }
      exts.shift();
    }
    
    return ext_data;
};


// helpers for backward-compatibility with the days of script_ids
// it sure would be nice to get rid of these...
function namespace_to_host(namespace) {
    var host_re = new RegExp(u.escape_re('^' + _DEFAULT_HOSTS_PATH));
    var acre_host_re = new RegExp(u.escape_re('^' + _DEFAULT_ACRE_HOST_PATH));
    
    if (host_re.test(namespace)) {
        namespace = namespace.replace(host_re, "");
        if (acre_host_re.test(namespace)) {
            namespace = namespace.replace(acre_host_re, "").replace(/^\//, "");
        }
    } else {
        namespace = _DELIMITER_PATH + namespace;
    }
    return namespace.split("/").reverse().join(".");
}

function host_to_namespace(host) {
    var path = null;
    
    var host_parts = host.split(".");
    var trailing_host_part = host_parts.pop();
    if (trailing_host_part === _DELIMITER_PATH) {
        path = "/" + host_parts.reverse().join("/");
    } else if (trailing_host_part === '') {
        path = _DEFAULT_HOSTS_PATH + "/" + host_parts.reverse().join("/");
    } else {
        host_parts.push(trailing_host_part);
        path = _DEFAULT_HOSTS_PATH + _DEFAULT_ACRE_HOST_PATH + "/" + host_parts.reverse().join("/");
    }
    
    return path;
}

function split_script_id(script_id) {
    var namespace = script_id.split('/');
    var f = namespace.pop();
    namespace = namespace.join('/');
    return [namespace, f];
}

function req_path_to_script_id(req_path) {
    var [host, script] = decompose_req_path(req_path);
    script = script.split("/").pop();
    return host_to_namespace(host) + "/" + script;
}


// random number generator used in CSRF protection
function get_random() {
    return Math.floor(Math.random()*10e15).toString();
};

function get_csrf_secret(that) {
    if (!_request.csrf_secret) {
        var key = _ks.get_key("csrf", _request.app_project);
        if (key && key[1]) {
            _request.csrf_secret = key[1];
        } else {
            _request.csrf_secret = get_random();
            _ks.put_key("csrf", _request.app_project, null, _request.csrf_secret);
        }
    }
    return _request.csrf_secret;
};


//--------------------------------- syslog --------------------------------------------

var _syslog = function(level, mesg, event_name) {
    if (typeof event_name == 'undefined') event_name = null;
    if (typeof mesg == 'undefined' || mesg === null) mesg = "undefined";
    _hostenv.syslog(level, event_name, mesg);
};

var syslog   = function (mesg, event_name) { return _syslog("INFO",mesg,event_name); };
syslog.debug = function (mesg, event_name) { return _syslog("DEBUG",mesg,event_name); };
syslog.info  = function (mesg, event_name) { return _syslog("INFO",mesg,event_name); };
syslog.warn  = function (mesg, event_name) { return _syslog("WARN",mesg,event_name); };
syslog.error = function (mesg, event_name) { return _syslog("ERROR",mesg,event_name); };

acre.syslog = syslog;

//--------------------------------- dev mode ------------------------------------------

/**
 *  this should be enabled for development only!
 */
var _dev = {};

_dev.test_internal = function (ename) {
    return _hostenv.dev_test_internal(ename);
};
_dev.syslog = syslog;

// developer entrypoints are invisible by default
if (_hostenv.ACRE_DEVELOPER_MODE) {
    acre._dev = _dev;
}

// ------------------------------- acre ----------------------------------------

acre.version = new String(_request.version);

/**
 *   begin the http response.
 *
 *   The Content-type header, if specified, is handled specially.
 *   If it is any text/* type, the charset specified in the header will
 *   be used to encode the response.  If no charset= parameter is given,
 *   the response will be encoded as UTF-8, and "; charset=utf-8" is added
 *   to the Content-type header before sending the response.
 *   <br/>
 *   For content types other than text/* the text is always encoded as UTF-8.
 *
 *   @param status is the response status (e.g. 200)
 *   @param headers is a javascript object
 */
acre.start_response = function (status, headers) {

    if (typeof status == 'undefined') {
        status = 200;
    } else if (typeof status == 'string') {
        status = parseInt(status, 10);
    } else if (typeof status == 'number') {
        acre.response.status = status;
    } else {
        throw new Error('acre.start_response: status "' + status.toSource() + '" must be an integer or string');
    }

    if (typeof headers == 'object') {
        // stupid merge - arguments override existing headers
        // some headers are patched in later though...
        u.extend(acre.response.headers, headers);
    }
};

/**
 *   add output to the http response.  multiple arguments will be appended.
 *   @param str string or markup to append
 */
acre.write = function () {
    u.each(arguments, function(i, arg) {
        // XXX - shouldn't have handler-specific code here
        if (typeof arg == 'object' && arg !== null &&
            (typeof arg.toMarkupList === 'function' || typeof arg.toMarkup === 'function')) {
            arg = acre.markup.stringify(arg);
        }

        if (typeof arg != 'string' && !(arg instanceof Binary))
            arg = '' + arg;

        _hostenv.write(arg);        
    });
};

/**
 * exit the current acre request.<br/><br/>
 *
 * this works by throwing an exception,
 * that exception propagates across acre.load* calls.<br/><br/>
 *
 * Currently, user code can catch that exception.
 * we may wish to change this behavior since most people expect
 * to catch errors rather than normal termination.<br/><br/>
 *
 * if an acre script exits before calling <b>acre.start_response()</b>,
 * an error response is generated.
 */
acre.exit = function () {
    throw new _hostenv.AcreExitException();
};

acre.wait = function(millis) {
    _hostenv.do_wait(millis);
}

// ------------------------------- acre.host -----------------------------------------

if (_request.server.indexOf("jetty") > -1) {
    var server_type = "acre_server";
} else if (_request.server.indexOf("app engine") > -1) {
    var server_type = "appengine";
} else {
    var server_type = new String(_request.server);
}

acre.host = {
    protocol : _request.server_protocol,
    name : _request.server_host_base,
    dev_name : _request.server_host,
    port : _request.server_port,
    server : server_type
};


//-------------------------------- acre.request -----------------------------------------

acre.request = {
    server_name : _request.request_server_name,
    server_port : _request.request_server_port,
    url : _request.request_url,
    base_url : _request.request_base_url,
    app_url : _request.request_app_url,
    protocol : _request.server_protocol, // XXX: how do we get the request protocol?
    method : _request.request_method,
    headers : _request.headers,
    base_path : _request.request_path_info.replace(new RegExp(u.escape_re(_request.path_info)+"$"), ""),    

    // these can be re-set by acre.route
    body : _request.request_body,
    path_info : _request.path_info,
    query_string : _request.query_string,
    
    // set in request params
    params: {},
    body_params: {},
    
    start_time : _request.request_start_time,
    cookies : {},
    skip_cache : false
};

// if we get 'cache-control: no-cache' in the request headers (i.e., shift-reload),
// then set a flag that appfetchers can key off of.
// some browsers will by default issue this header for *all* XHRs, however,
// which can make requests very slow. the work-around is that if an 
// x-acre-cache-control header is present, then we do not set the flag
if ('cache-control' in acre.request.headers && !('x-acre-cache-control' in acre.request.headers)) {
    if (/no-cache/.test(acre.request.headers['cache-control'])) {
        acre.request.skip_cache = true;
    }
}

// Deal with old metaweb cookie weirdness (TODO - destroy this asap)
// Close to avoid leaking variables into the scope
_request.mwlt_mode = false;
(function () {
    var _MWAUTH_HOSTS = _hostenv.ACRE_ALLOW_MWAUTH_HOST_SUFFIX.split(' ');
     var ok = false;
     var host = acre.request.server_name;
     for (var a in _MWAUTH_HOSTS) {
         var h = _MWAUTH_HOSTS[a];
         if (host.match(h+'$')){ // && !(host.match('\.acrenet\.metaweb\.com$'))) {
             ok = true;
             break;
         }
     }
     if (ok)
         _request.mwlt_mode = true;

    for (var name in _request.cookies) {
        if (name == 'metaweb-user-info') {
            var user_cookie = _request.cookies['metaweb-user-info'].value;

            function get_part(key) {
                var cookie_part = new RegExp('\\|('+key+')_(.*?)\\|');
                var match = cookie_part.exec(user_cookie);
                return decodeURIComponent(match[2]);
            }

            acre.request.user_info = {
               guid : get_part('g'),
               id : get_part('p'),
               name : get_part('u')
            };
        } else if (name == 'metaweb-user') {
            if (!ok) {
                syslog.debug("deleting metaweb-user cookie");
                delete _request.cookies['metaweb-user'];
            } else {
                acre.request.cookies[name] = _request.cookies[name].value;
            }
        } else {
            acre.request.cookies[name] = _request.cookies[name].value;
        }
    }

})();

/**
 * reset acre.request.query_string & body
 * 
 * Also parse params into objects for easy look-up:
 *   query_string --> acre.request.params
 *   body --> acre.request.body_params
 */
function set_request_params() {
    var query_string = acre.request.query_string;
    var body = acre.request.body;
    
    try {
        acre.request.params = (typeof query_string == 'string') ? acre.form.decode(query_string) : {};
        acre.request.body_params = (typeof body == 'string' && typeof acre.request.headers['content-type'] == 'string' && acre.request.headers['content-type'].match(/^application\/x-www-form-urlencoded/)) ? acre.form.decode(body) : {};
    } catch (e) {
        acre.request.params = {};
        acre.request.body_params = {};
        console.warn("Invalid request: parameters were not properly encoded");
    }
}


// ------------------------------- acre.respose -----------------------------------------

function AcreResponse() {
    this.status = 200;
    this.headers = {};
    this.cookies = {};
}

// Variables set by internal private helpers
var AcreResponse_availability = false;
var AcreResponse_max_age = 0;
var AcreResponse_vary_cookies = {};
var AcreResponse_session = acre.request.cookies['ACRE_SESSION'];
var AcreResponse_set_session_cookie = false;

var AcreResponse_validate_header = function(name, value) {
    // according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2
    var http_token_re = /[-<KEY> // ']

    // check that name and value are valid
    if (!http_token_re.test(name)) {
        throw new Error('illegal HTTP header name');
    }

    name = name.toLowerCase();

    if (/[\r\n]/.test(value)) {
        throw new Error('newline is illegal in HTTP header value');
    }
};

// AcreResponse methods are exposed to user scripts as acre.response.*
AcreResponse.prototype.set_header = function (name, value) {
    name = name.toLowerCase();
    AcreResponse_validate_header(name, value);
    this.headers[name] = value;
};

AcreResponse.prototype.add_header = function (name, value) {
    name = name.toLowerCase();
    AcreResponse_validate_header(name, value);

    // RFC2616 sec 4.2: It MUST be possible to combine the multiple
    //   header fields into one "field-name: field-value" pair, without
    //   changing the semantics of the message, by appending each
    //   subsequent field-value to the first, each separated by a comma.
    if (!(name in this.headers)) {
        this.headers[name] = value;
    } else if (u.isArray(this.headers[name])) {
        this.headers[name].push(value);
    } else {
        this.headers[name] = [this.headers[name], value];
    }
};

AcreResponse.prototype.set_header_default = function (name, value) {
    name = name.toLowerCase();
    if (name in this.headers) {
        return;
    }
    this.set_header(name, value);
};

AcreResponse.prototype.clear_cookie = function (name, options) {
    if (typeof options == 'undefined') {
        options = {};
    }
    var clear_cookie_options = { domain:1, path:1, secure:1 };
    var newopts = {};
    for (k in clear_cookie_options) {
        if (k in options) {
            newopts[k] = options[k];
        }
    }
    newopts.max_age = 0;
    AcreResponse_vary_cookies[name] = 1;
    this.set_cookie(name, '', newopts);
};

// options is an object with valid properties being  domain, path, secure, max_age, expires
// max_age is preferred to expires, it gives a lifetime in seconds
// max_age=0 causes the cookie to be cleared
// expires is accepted as a string or a javascript Date object
// max_age causes an expires attribute to be generated too for IE's sake.
AcreResponse.prototype.set_cookie = function (name, value, options) {
    var valid_cookie_options = { domain:1, path:1, secure:1, max_age:1, expires:1 };

    if (typeof options == 'undefined') {
        options = {};
    }

    if (typeof options.max_age != 'undefined' && typeof options.max_age != 'number') {
        throw new Error('set_cookie() max_age must be a number');
    }

    // TODO should check value for non-cookie ok strings
    var cookie = { name: name, value: value };

    for (var k in options) {
        if (!(k in valid_cookie_options)) {
            throw new Error('Invalid cookie option "' + k + '"');
        }
        cookie[k] = options[k];
    }

    // the acre java code understands max-age= but not expires=
    // the jetty cookie manager converts max-age to expires in the response.
    if (typeof cookie.max_age == 'undefined') {
        // calculate max-age= from expires= if present
        if (typeof cookie.expires != 'undefined') {
            var d = cookie.expires;
            if (!(d instanceof Date)) {
                d = Date.parse(d);
            }

            // find the time from now until the expires= time
            var delta = d.getTime() - (new Date()).getTime();
            if (delta <= 0) {
                cookie.max_age = 0;
            } else {
                cookie.max_age = Math.ceil(delta / 1000.0);
            }
        }
    } else if (typeof cookie.expires != 'undefined') {
        // complain if both max-age= and expires= are given
        console.warn('Cookie "max_age" attribute overides "expires" attribute');
    }

    AcreResponse_vary_cookies[name] = 1;
    this.cookies[name] = cookie;
};

AcreResponse.prototype.set_cache_policy = function (policy) {
    switch (policy) {
        case 'fresh':
            AcreResponse_max_age = 0;
            AcreResponse_availability = false;
            break;
        case 'fast':
            AcreResponse_max_age = 3600;
            AcreResponse_availability = true;
            break;
    }
};

AcreResponse.prototype.set_policy =
    AcreResponse.prototype.set_cache_policy; // deprecated

// these AcreResponse_* functions are called during _hostenv.finish_response()
var AcreResponse_generate_date = function (d) {
    function padz(n) {
        return n > 10 ? n : '0' + n;
    }

    var MONTH_NAMES = [
        "January", "February", "March",
        "April", "May", "June",
        "July", "August", "September",
        "October", "November", "December"
    ];

    var WEEKDAY_NAMES = [
        "Sunday", "Monday", "Tuesday",
        "Wednesday", "Thursday", "Friday",
        "Saturday"
    ];

    var dayname = WEEKDAY_NAMES[d.getUTCDay()].slice(0,3);
    var day = padz(d.getUTCDate());
    var monthname = MONTH_NAMES[d.getUTCMonth()].slice(0,3);
    var year = d.getUTCFullYear();
    var hour = padz(d.getUTCHours());
    var minutes = padz(d.getUTCMinutes());
    var seconds = padz(d.getUTCSeconds());

    return dayname+', '+day+' '+monthname+' '+year+' '+hour+':'+ minutes +':'+ seconds+' UTC';
};

var AcreResponse_set_expires = function (that) {
    var expires = new Date(acre.request.start_time.getTime()+(AcreResponse_max_age*1000));
    var d = AcreResponse_generate_date(expires);

    that.set_header_default('Expires', d);
};

var AcreResponse_set_last_modified = function (that) {
    var d = AcreResponse_generate_date(acre.request.start_time);
    that.set_header_default('Last-Modified', d);
};

var AcreResponse_set_cache_control = function (that) {
    var cc = '';

    if (AcreResponse_availability == true) {
        cc += 'public';
        cc += ', no-cache="Set-Cookie"';

        cc += ', max-age='+AcreResponse_max_age;
        for (var a in AcreResponse_vary_cookies) {
            cc += ', maxage-vary-cookie="'+AcreResponse_max_age+'|'+a+'"';
        }

    } else {
        cc += 'private, no-cache, max-age=0';
    }

    that.set_header_default('cache-control', cc);
};

var AcreResponse_set_vary = function (that) {
    if (acre.response.headers.vary){
        var vary = acre.response.headers.vary.split(", ");
    } else {
        var vary = [];
    }
    
    for (key in AcreResponse_vary_cookies) {
        vary.push('Cookie');
        break;
    }

    if (vary.length > 0) {
        acre.response.set_header('Vary', vary.join(', '));
    }
};

var AcreResponse_set_metaweb_vary = function (that) {
    var vary = [];
    for (var c in AcreResponse_vary_cookies.__count__) {
        vary.push('Cookie['+c+']');
    }

    if (vary.length > 0) {
        acre.response.set_header_default('x-metaweb-vary', vary.join(', '));
    }
};

var AcreResponse_get_session = function(that) {
    if (typeof AcreResponse_session === 'undefined') {
        AcreResponse_session = get_random();
        AcreResponse_set_session_cookie = true;
    }
    return AcreResponse_session;
};

var AcreResponse_set_session = function(that) {
    if (AcreResponse_set_session_cookie) {
        that.set_cookie("ACRE_SESSION", AcreResponse_session);
    }
};

acre.response = null;


// --------------------------- acre.errors ---------------------------------

/*
 * Setup the acre.error object in case the request is for an error page
 */
if (_request.error_info) {
    acre.error = _request.error_info;

    // error-specific info is passed as a JSON string - decode it here
    var info_json = acre.error.info_json;
    acre.error.info = _json.parse('['+acre.error.info_json+']')[0];
    delete acre.error.info_json;
}

acre.errors = {};

/**
 *  exception class to allow bailing out early, e.g.
 *   after responding quickly with an error code.
 *  this api is likely to change so this exception
 *  should be thrown by calling acre.exit(), not
 *  by thowing it directly.  it is documented here
 *  because developers might notice this exception
 *  getting caught in their javascript "catch" blocks.
 *  if this occurs, the best thing is probably to
 *  re-throw the exception.
 */
acre.errors.AcreRouteException = function () {};
acre.errors.AcreRouteException.prototype = new Error('acre.route');

acre.errors.AcreExitException = function () {};
acre.errors.AcreExitException.prototype = new Error('acre.exit');
_hostenv.AcreExitException = acre.errors.AcreExitException;

acre.errors.URLError = function (message) {
    this.message = message;
};
acre.errors.URLError.prototype = new Error('acre.errors.URLError');
acre.errors.URLError.prototype.name = 'acre.errors.URLError';
_hostenv.URLError = acre.errors.URLError;

acre.errors.URLTimeoutError = function (message) {
    this.message = message;
};
acre.errors.URLTimeoutError.prototype = new acre.errors.URLError('acre.errors.URLTimeoutError');
acre.errors.URLTimeoutError.prototype.name = 'acre.errors.URLTimeoutError';
_hostenv.URLTimeoutError = acre.errors.URLTimeoutError;

_topscope.AcreExitException = acre.errors.AcreExitException; // deprecated
_topscope.URLError = acre.errors.URLError; // deprecated

_hostenv.Error = Error;


// ------------------------------- acre.environ -----------------------------------

// deprecated
function set_environ() {
    // avoid deprecation warnings
    // if already set up (acre.route)
    if (acre.environ) {
        delete acre.environ.query_string;
        delete acre.environ.params;
        delete acre.environ.body_params;
        delete acre.environ.path_info;
        delete acre.environ.script_name;
        delete acre.environ.script_id;
        delete acre.environ.script_namespace;
        delete acre.environ.user_info;
        delete acre.environ;
    }
    
    acre.environ = {
        version: _request.version,
        script_name: _request.script_name,
        script_id: _request.script_id,
        script_namespace: _request.script_namespace,
        path_info: _request.path_info,
        query_string: _request.query_string,
        headers: _request.headers,
        cookies: _request.cookies,
        server_name: _request.server_name,
        server_host: _request.server_host,
        server_host_base: _request.server_host_base,
        server_port: _request.server_port,
        server_protocol: _request.server_protocol,
        request_body: _request.request_body,
        request_method: _request.request_method,
        request_path_info: _request.request_path_info,
        request_server_name: _request.request_server_name,
        request_server_port: _request.request_server_port,
        request_start_time: _request.request_start_time,
        request_url: _request.request_url,
        request_app_url: _request.request_app_url,
        request_base_url: _request.request_base_url,
        freebase_service_url: _request.freebase_service_url,
        freebase_site_host: _request.freebase_site_host
    };
    acre.environ.params = acre.request.params;
    acre.environ.body_params = acre.request.body_params;
    
    if (acre.request.user_info) {
        acre.environ.user_info = acre.request.user_info;
    }
};


// ------------------------------------ JSON ------------------------------

_topscope.JSON = {
    stringify: function(obj, resolver, space) {
        try {
            return _json.stringify(obj, resolver, space);
        } catch (e) {
          var message = e.message.indexOf("JSON.stringify:") == 0
                          ? e.message
                          : e.message.replace(/^[^:]*:\s*/, '');
            throw new Error(message);
        }
    },
    parse: function(str, reviver) {
        try {
            if (typeof reviver !== "undefined" && reviver != null) {
                console.warn("Acre currently ignores 'reviver' in JSON.parse()");
            }

            return _json.parse(str);
        } catch (e) {
            // Everything after the first :, if present
            throw new Error(e.message.replace(/^[^:]*:\s*/, ''));
        }
    }
};


// ------------------------------ xml/html parsers ----------------------------

if (typeof acre.xml == 'undefined') {
    acre.xml = {};
}

if (typeof acre.html == 'undefined') {
    acre.html = {};
}

/**
 *  Parse a string containing XML into a DOM Document object. <br/><br/>
 *
 *  The returned Document object complies to the W3C EcmaScript bindings for
 *  the DOM.
 *
 *  @param xml_str the XML string to be parsed
 */
acre.xml.parse = function(xml_str) {
    return _domparser.parse_string(xml_str, "xml");
};

/**
 *  Parse a string containing XML into a DOM Document object and support namespaces. <br/><br/>
 *
 *  The returned Document object complies to the W3C EcmaScript bindings for
 *  the DOM.
 *
 *  @param xml_str the XML string to be parsed
 */
acre.xml.parseNS = function(xml_str) {
    return _domparser.parse_ns_string(xml_str, "xml");
};

/**
 *  Parse a string containing HTML into a DOM Document object. <br/><br/>
 *
 *  The returned Document object complies to the W3C EcmaScript bindings for
 *  the DOM.
 *
 *  @param html_str the HTML string to be parsed
 */
acre.html.parse = function(html_str) {
    return _domparser.parse_string(html_str, "html");
};


// ------------------------ acre.hash ------------------------------

if (typeof acre.hash == 'undefined') {
    acre.hash = {};
}

acre.hash.hex_sha1 = function (s) {
    return _hostenv.hash("SHA-1", s, true);
};

acre.hash.b64_sha1 = function (s) {
    return _hostenv.hash("SHA-1", s, false);
};

acre.hash.hex_md5 = function (s) {
    return _hostenv.hash("MD5", s, true);
};

acre.hash.b64_md5 = function (s) {
    return _hostenv.hash("MD5", s, false);
};

acre.hash.hex_hmac_sha1 = function (key, data) {
    return _hostenv.hmac("HmacSHA1", key, data, true);
};

acre.hash.b64_hmac_sha1 = function (key, data) {
    return _hostenv.hmac("HmacSHA1", key, data, false);
};

acre.hash.hex_hmac_md5 = function (key, data) {
    return _hostenv.hmac("HmacMD5", key, data, true);
};

acre.hash.b64_hmac_md5 = function (key, data) {
    return _hostenv.hmac("HmacMD5", key, data, false);
};


// ------------------------------------------------------------------------

// used by the keystore and the cache to partition data based on the app_guid

_request.app_project = null;

// ------------------------ keystore --------------------------------------

if (typeof acre.keystore == 'undefined') {
    acre.keystore = {};
}

acre.keystore.get = function (name) {
    if (_request.app_project !== null) {
        return _ks.get_key(name, _request.app_project);
    } else {
        return null;
    }
};

acre.keystore.keys = function () {
    if (_request.app_project !== null) {
        return _ks.get_keys(_request.app_project);
    } else {
        return null;
    }
};

acre.keystore.remove = function (name) {
    if (_request.app_project !== null) {
        _ks.delete_key(name, _request.app_project);
    }
};

if (_request.trusted) { 
    acre.keystore.put = function (name, token, secret) {
        if (_request.app_project !== null) {
            return _ks.put_key(name, _request.app_project, token, secret);
        } else {
            return null;
        }
    };
}

//------------------------ cache --------------------------

var cache_scope = {};
cache_scope.syslog = syslog;
cache_scope.cache = _cache;
cache_scope.acreboot = _topscope;
cache_scope.request = _request;
_hostenv.load_system_script('cache.js', cache_scope);
cache_scope.augment(acre);

//------------------------ datastore --------------------------

if (_request.trusted && _datastore) { // the _datastore object won't be available in all environments so we need to check first
    var store_scope = {};
    store_scope.syslog = syslog;
    store_scope.store = _datastore;
    _hostenv.load_system_script('datastore.js', store_scope);
    store_scope.augment(acre);
}

//------------------------ taskqueue --------------------------

if (_request.trusted && _taskqueue) { // the _taskqueue object won't be available in all environments so we need to check first
    var queue_scope = {};
    queue_scope.syslog = syslog;
    queue_scope.queue = _taskqueue;
    _hostenv.load_system_script('taskqueue.js', queue_scope);
    queue_scope.augment(acre);
}

//------------------------ mailer --------------------------

if (_request.trusted && _mailer) { // the _mailer object won't be available in all environments so we need to check first
    var mailer_scope = {};
    mailer_scope.syslog = syslog;
    mailer_scope.mailer = _mailer;
    _hostenv.load_system_script('mailservice.js', mailer_scope);
    mailer_scope.augment(acre);
}

// ------------------------------ urlfetch -----------------------------------

/**
 *   allows an acre script to fetch data from an external url.<br/><br/>
 *
 *   this may not be part of the standard api, we will probably want to
 *   restrict which urls are available to prevent abuse.<br/><br/>
 *
 *   note that response.body is a javascript string, which means it
 *   has already been decoded.  there are bugs in this process,
 *   because we don't sniff for <meta http-equiv="content-type"...> here.<br/><br/>
 *
 *   <pre>
 *   returns a response object, e.g.:
 *     {
 *       status: 200,
 *       headers: {
 *          "content-length": "14",
 *          "content-type": "text/plain; charset=UTF-8",
 *       },
 *       content_type: "text/plain",
 *       body: "hocus.\npocus.\n"
 *    }
 *    </pre><br/><br/>
 *
 *  all header names are canonicalized to lowercase
 *
 */

var _system_urlfetch = function(url,method,headers,content,sign) {
    return _urlfetch(true,url,method,headers,content,sign);
};
var _system_async_urlfetch = function(url,method,headers,content,sign) {
    return _urlfetch(true,url,method,headers,content,sign, _hostenv.urlOpenAsync);
};

// acre.urlfetch is disabled in error scripts.  the reason is
//  that error scripts are given a special extension on the
//  original script's time quota.  they can't be allowed
//  to re-enter the server or they would be able to pass on
//  that time quota to another request.
// this could be relaxed to allow users to call services
//  other than acre services.
acre.async = {};
if (!acre.error) {
    acre.urlfetch = function(url,method,headers,content,sign) {
        return _urlfetch(false,url,method,headers,content,sign);
    };
    acre.async.urlfetch = function(url, method, headers, content, sign) {
        return _urlfetch(false,url,method,headers,content,sign,_hostenv.urlOpenAsync);
    };
    acre.async.wait_on_results = function (timeout) {
        _hostenv.async_wait(timeout);
    };
} else {
    acre.urlfetch = function(url,method,headers,content,sign) {
        throw new Error('acre.urlfetch() is restricted in error scripts');
    };
    acre.async.urlfetch = function(url, method, headers, content, sign) {
        throw new Error('acre.async.urlfetch() is restricted in error scripts');
    };
    acre.async.wait_on_results = function () {
        throw new Error('acre.async.wait_on_results() is restricted in error scripts');
    };
}

/*
 * Fetches an HTTP URL.
 *
 * Parameters:
 *
 *  - system: whether or not this is a fetch made by acre directly or by the user script [required]
 *  - url: the URL to fetch [required]
 *  - method: the HTTP method to use to fetch it [optional, defaults to GET]
 *  - headers: the HTTP headers to pass along with the request [optional]
 *  - content: the HTTP payload of the request [optional]
 *  - sign: whether or not sign the request (using oauth) [optional]
 *
 * NOTE: sign can be either a boolean or an object. If a boolean
 * the oauth subsystem will look for the right key based on the domain
 * name of the request being made. If it's an object it must contain the key/secret
 * pair that identifies the application against the oauth provider and
 * must be in the form:
 *
 *     {
 *       "consumerKey" : key,
 *       "consumerSecret" : secret
 *     }
 */
var _urlfetch = function (system, url, options_or_method, headers, content, sign, _urlopener) {
    _urlopener = _urlopener || _hostenv.urlOpen;

    // exception generation would be nice here
    if (typeof url != 'string' || !url.length) {
        throw new acre.errors.URLError("'url' argument (1st) to acre.urlfetch() must be a string");
    }
    
    var method;
    var response_encoding = null;
    var callback;
    var errback;
    var timeout;
    var bless;
    var no_redirect;
    
    if (options_or_method && typeof options_or_method === 'object') {
        if (typeof headers !== 'undefined' || typeof content !== 'undefined' || typeof sign !== 'undefined') {
            throw new acre.errors.URLError("'options' argument (2nd) to acre.urlfetch() must not be followed by extra arguments");
        }

        method = options_or_method.method;
        headers = options_or_method.headers;
        content = options_or_method.content;
        sign = !!options_or_method.sign;
        response_encoding = options_or_method.response_encoding;
        callback = options_or_method.callback || function (res) { };
        errback = options_or_method.errback || function (res) { throw res; };
        var i =  parseInt(options_or_method.timeout, 10) ;
        timeout = (i > 0) ? i : undefined;
        bless = !!options_or_method.bless;
        no_redirect = options_or_method.no_redirect;
    } else if (options_or_method) {
        method = options_or_method;
    }

    if (typeof sign !== 'undefined' && typeof sign !== 'boolean' && typeof sign !== 'object') {
        throw new acre.errors.URLError("'sign' argument (5th) or option to acre.urlfetch() must be a boolean or an object");
    }

    if (typeof method != 'string') method = 'GET';
    if (typeof headers != 'object') headers = {};
    if (typeof content != 'string' && !(content instanceof Binary)) content = '';
    if (typeof response_encoding != 'string') response_encoding = 'utf-8';
    if (typeof no_redirect != 'boolean') no_redirect = false;
    
    // lowercase all the header names so we don't have to worry about
    // case sensitivity
    for (var hk in headers) {
        var hkl = hk.toLowerCase();
        if (hk == hkl) continue;
        var hkv = headers[hk];
        delete headers[hk];
{'fraction_non_alphanumeric': 0.15763169972971594, 'fraction_numerical': 0.0033530521699219745, 'mean_word_length': 2.444146834109072, 'pattern_counts': {'":': 4, '<': 58, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 26}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4414749', 'n_tokens_mistral': 24472, 'n_tokens_neox': 23725, 'n_words': 6213}