<commit_before>var util = require('util'),
    path = require('path'),
    j = require('path').join,
    i = function(obj, depth, showAll) { return util.inspect(obj, showAll, typeof depth === 'number' ? depth : 1); },
    async = require(j(process.env.LK_SCRIPTS_ROOT, 'node_modules/async'));

function uuid() { // helper
    var id = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
        var r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8); return v.toString(16); }).toUpperCase();
    return id;
}

var debugLevel = 1;
var debugThreshold = 1;
var logSessionLifetime = true;

function log(/*level, msg, arguments*/) {
    // the smaller logLevel the more important the message
    var args = Array.prototype.slice.call(arguments);
    var logLevel = typeof args[0] === 'number' ? args.shift() : 1;
    if (logLevel > debugThreshold) return;
    console.log.apply(console, args);
}

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// session actions, when messages come in they specify in
// their "action" parameter what they want to do. This is
// the table that defines what functions are behind those
// actions.
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
var sessionActions = {

    reportServices: function(sessionServer, connection, msg) {
        connection.send({
            action: msg.action + 'Result',
            inResponseTo: msg.messageId,
            data: {services: Object.keys(sessionServer.getActions())}
        });
    },

    registerClient: function(sessionServer, connection, msg) {
        // Adds a lively2lively session object for the client identified with
        // msg.data.id (or reuses the old session if client is already registered).
        // Note that there is a difference between the session object created
        // here and the (HTTP/websocket) connection object used to send/receive
        // messages: The session can persist even though the underlaying
        // connection may change or be closed. Session objects have the structure: {
        //     timeOfCreation: NUMBER,    -- timestamp of session creation, reported by client
        //     timeOfRegistration: NUMBER -- timestamp of session registration, attached by server
        //     lastActivity: NUMBER,      -- timestamp of last reported activity
        //     user: STRING,              -- name of the session, not unique
        //     worldURL: STRING           -- the URL of the connected "world"
        //     id: STRING                 -- unique, identifies the client, looks like 'client-session:E25F3CCC-F82D-41F4-B3E7-CDA398BD7D2D'
        //     remoteAddress: STRING      -- ip
        // }
        var sessions = sessionServer.getLocalSessions()[sessionServer.id()],
            session = sessions[msg.sender],
            isNewSession = false;

        if (!session) {
            isNewSession = true;
            session = sessions[msg.sender] = {}
        }

        util._extend(session, msg.data);
        util._extend(session, {
            remoteAddress: connection.remoteAddress,
            timeOfRegistration: new Date().getTime()
        });

        connection.id = msg.data.id;
        connection.on('close', function() {
            // remove the session registration after the connection closes
            // give it some time because the connection might be just flaky and
            // the client might come back
            if (!sessions[msg.sender]) return; // unregistered
            if (!sessionServer.inactiveSessionRemovalTime) return;
            setTimeout(function() {
                var newConnection = sessionServer.websocketServer.getConnection(msg.sender);
                if (!newConnection) {
                    log(2, '%s removes local session of %s', sessionServer, msg.sender);
                    sessionServer.removeLocalSessionOf(msg.sender);
                    if (logSessionLifetime) {
                        console.log('l2l deregistration "%s" %s %s %s',
                            session.user, session.worldURL, session.id, session.remoteAddress);
                    }
                }
            }, sessionServer.inactiveSessionRemovalTime);
        });

        if (logSessionLifetime && isNewSession) {
            console.log('l2l registration "%s" %s %s %s',
                session.user, session.worldURL, session.id, session.remoteAddress);
        }

        connection.send({
            action: msg.action + 'Result',
            inResponseTo: msg.messageId,
            data: {success: true, tracker: {id: sessionServer.id()}}
        });
    },

    unregisterClient: function(sessionServer, connection, msg) {
        var session = sessionServer.removeLocalSessionOf(msg.sender);
        if (logSessionLifetime) {
            console.log('l2l deregistration "%s" %s %s %s',
                session.user, session.worldURL, session.id, session.remoteAddress);
        }
        connection.send({
            action: msg.action + 'Result',
            inResponseTo: msg.messageId, data: {success: true}});
        connection.close();
    },

    initServerToServerConnect: function(sessionServer, connection, msg) {
        var url = msg.data.url;
        sessionServer.serverToServerConnect(url, function(err, remoteClient) {
            if (err) console.error(err);
            connection.send({action: 'initServerToServerConnectResult', inResponseTo: msg.messageId, data: {success: !err}});
        });
    },

    initServerToServerDisconnect: function(sessionServer, connection, msg) {
        var remoteURLs = Object.keys(sessionServer.serverToServerConnections);
        sessionServer.removeServerToServerConnections();
        connection.send({action: msg.action + 'Result', inResponseTo: msg.messageId, data: {success: true, message: 'Connections to ' + remoteURLs.join(', ') + ' closed'}});
    },

    getSessions: function(sessionServer, connection, msg) {
        // send the sessions accessible from this tracker. includes local sessions,
        // reported sessions
        sessionServer.getSessionList(msg.data.options, function(sessions) {
            connection.send({
                action: msg.action,
                inResponseTo: msg.messageId,
                data: sessions
            });
        })
    },

    reportSessions: function(sessionServer, connection, msg) {
        // another tracker is reporting its sessions to sessionServer.
        // sessionServer is acting as a "lively2lively central"
        if (!msg.data || !msg.data.trackerId) {
            console.error('%s got reportSession request without id: ', sessionServer, msg);
            return;
        }
        var id = connection.id = msg.data.trackerId;
        connection.on('close', function() {
            if (!sessionServer.inactiveSessionRemovalTime) return;
            setTimeout(function() {
                var newConnection = sessionServer.websocketServer.getConnection(msg.sender);
                if (!newConnection) {
                    log(2, '%s removes reported session of %s', sessionServer, id);
                    delete sessionServer.trackerData[id];
                }
            }, sessionServer.inactiveSessionRemovalTime);
        });
        sessionServer.trackerData[id] = {sessions: msg.data[id]};
        connection.send({action: msg.action + 'Result', inResponseTo: msg.messageId, data: {success: true, message: 'Sessions added to ' + sessionServer}});
    },

    reportActivity: function(sessionServer, connection, msg) {
        // lively session sends infos about last user activity from time to time
         var sessions = sessionServer.getLocalSessions()[sessionServer.id()],
            session = sessions[msg.sender] = sessions[msg.sender] || {};
        session.lastActivity = msg.data.lastActivity;
        connection.send({action: msg.action + 'Result', inResponseTo: msg.messageId, data: {success: true}});
    }

}

var services = require("./LivelyServices").services;
util._extend(services, sessionActions);

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// SessionTracker
// The main server component for managing sessions. A
// session represents a onlinr connection of a lively
// world that can be used to communicate to/from that world
// The SessionTracker will manage the connections to Lively
// worlds and connections to other Lively servers (server-
// to-server connections). Those connections are used to
// a) route messages from one session to another (local
// seends as well as remote sends requiring server-to-server
// routing are supported).
// Also, certain actions (getSessions, reportActivity) are
// directly processed by the tracker and do not require
// connections to other sessions.
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
var websockets = require(path.join(process.env.WORKSPACE_LK, 'core/servers/support/websockets'));<commit_msg>l2l: log l2l events, server side<commit_after>var util = require('util'),
    path = require('path'),
    j = require('path').join,
    i = function(obj, depth, showAll) { return util.inspect(obj, showAll, typeof depth === 'number' ? depth : 1); },
    async = require(j(process.env.LK_SCRIPTS_ROOT, 'node_modules/async'));

function uuid() { // helper
    var id = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
        var r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8); return v.toString(16); }).toUpperCase();
    return id;
}

var debugLevel = 1;
var debugThreshold = 1;
var logSessionLifetime = true;

function log(/*level, msg, arguments*/) {
    // the smaller logLevel the more important the message
    var args = Array.prototype.slice.call(arguments);
    var logLevel = typeof args[0] === 'number' ? args.shift() : 1;
    events.add.apply(events, args);
    if (logLevel > debugThreshold) return;
    console.log.apply(console, args);
}

var events = global.l2lEvents || (global.l2lEvents = {
    log: []
});

util._extend(events, {

    add: function(/*logMessage args*/) {
        var msg = util.format.apply(util, arguments);
        var entry = {timestamp: String(new Date()), message: msg};
        if (events.log.length > 1000) events.log = events.log.slice(-800);
        events.log.push(entry);
    }

})

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// session actions, when messages come in they specify in
// their "action" parameter what they want to do. This is
// the table that defines what functions are behind those
// actions.
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
var sessionActions = {

    reportServices: function(sessionServer, connection, msg) {
        events.add('[sender %s][reportServices]', msg.sender);
        connection.send({
            action: msg.action + 'Result',
            inResponseTo: msg.messageId,
            data: {services: Object.keys(sessionServer.getActions())}
        });
    },

    registerClient: function(sessionServer, connection, msg) {
        // Adds a lively2lively session object for the client identified with
        // msg.data.id (or reuses the old session if client is already registered).
        // Note that there is a difference between the session object created
        // here and the (HTTP/websocket) connection object used to send/receive
        // messages: The session can persist even though the underlaying
        // connection may change or be closed. Session objects have the structure: {
        //     timeOfCreation: NUMBER,    -- timestamp of session creation, reported by client
        //     timeOfRegistration: NUMBER -- timestamp of session registration, attached by server
        //     lastActivity: NUMBER,      -- timestamp of last reported activity
        //     user: STRING,              -- name of the session, not unique
        //     worldURL: STRING           -- the URL of the connected "world"
        //     id: STRING                 -- unique, identifies the client, looks like 'client-session:E25F3CCC-F82D-41F4-B3E7-CDA398BD7D2D'
        //     remoteAddress: STRING      -- ip
        // }
        var sessions = sessionServer.getLocalSessions()[sessionServer.id()],
            session = sessions[msg.sender],
            isNewSession = false;

        if (!session) {
            isNewSession = true;
            session = sessions[msg.sender] = {}
        }

        util._extend(session, msg.data);
        util._extend(session, {
            remoteAddress: connection.remoteAddress,
            timeOfRegistration: new Date().getTime()
        });

        connection.id = msg.data.id;
        connection.on('close', function() {
            // remove the session registration after the connection closes
            // give it some time because the connection might be just flaky and
            // the client might come back
            if (!sessions[msg.sender]) return; // unregistered
            if (!sessionServer.inactiveSessionRemovalTime) return;
            events.add('[connection to %s] closed', msg.sender);
            setTimeout(function() {
                var newConnection = sessionServer.websocketServer.getConnection(msg.sender);
                if (!newConnection) {
                    log(2, '%s removes local session of %s', sessionServer, msg.sender);
                    events.add('[connection to %s] l2l deregistration "%s" %s %s %s',
                        msg.sender, session.user, session.worldURL, session.id, session.remoteAddress);
                    sessionServer.removeLocalSessionOf(msg.sender);
                    if (logSessionLifetime) {
                        console.log('l2l deregistration "%s" %s %s %s',
                            session.user, session.worldURL, session.id, session.remoteAddress);
                    }
                }
            }, sessionServer.inactiveSessionRemovalTime);
        });

        if (logSessionLifetime && isNewSession) {
            console.log('l2l registration "%s" %s %s %s',
                session.user, session.worldURL, session.id, session.remoteAddress);
        }

        events.add('[sender %s][registerClient] l2l registration "%s" %s %s %s%s',
            msg.sender, session.user, session.worldURL, session.id, session.remoteAddress, isNewSession ? ' (new)' : '');

        connection.send({
            action: msg.action + 'Result',
            inResponseTo: msg.messageId,
            data: {success: true, tracker: {id: sessionServer.id()}}
        });
    },

    unregisterClient: function(sessionServer, connection, msg) {
        var session = sessionServer.removeLocalSessionOf(msg.sender);
        if (logSessionLifetime) {
            console.log('l2l deregistration "%s" %s %s %s',
                session.user, session.worldURL, session.id, session.remoteAddress);
        }
        events.add('[sender %s][unregisterClient] l2l deregistration "%s" %s %s %s',
                msg.sender, session.user, session.worldURL, session.id, session.remoteAddress);

        connection.send({
            action: msg.action + 'Result',
            inResponseTo: msg.messageId, data: {success: true}});
        connection.close();
    },

    initServerToServerConnect: function(sessionServer, connection, msg) {
        var url = msg.data.url;
        events.add('[sender %s][initServerToServerConnect] url: %s', msg.sender, url);
        sessionServer.serverToServerConnect(url, function(err, remoteClient) {
            if (err) console.error(err);
            connection.send({action: 'initServerToServerConnectResult', inResponseTo: msg.messageId, data: {success: !err}});
        });
    },

    initServerToServerDisconnect: function(sessionServer, connection, msg) {
        var remoteURLs = Object.keys(sessionServer.serverToServerConnections);
        sessionServer.removeServerToServerConnections();
        events.add('[sender %s][initServerToServerDisconnect]', msg.sender);
        connection.send({action: msg.action + 'Result', inResponseTo: msg.messageId, data: {success: true, message: 'Connections to ' + remoteURLs.join(', ') + ' closed'}});
    },

    getSessions: function(sessionServer, connection, msg) {
        // send the sessions accessible from this tracker. includes local sessions,
        // reported sessions
        events.add('[sender %s][getSessions]', msg.sender);
        sessionServer.getSessionList(msg.data.options, function(sessions) {
            connection.send({
                action: msg.action,
                inResponseTo: msg.messageId,
                data: sessions
            });
        })
    },

    reportSessions: function(sessionServer, connection, msg) {
        // another tracker is reporting its sessions to sessionServer.
        // sessionServer is acting as a "lively2lively central"
        if (!msg.data || !msg.data.trackerId) {
            console.error('%s got reportSession request without id: ', sessionServer, msg);
            return;
        }

        var id = connection.id;
        if (!id) {
            id = connection.id = msg.data.trackerId;
            connection.on('close', function() {
                if (!sessionServer.inactiveSessionRemovalTime) return;
                setTimeout(function() {
                    var newConnection = sessionServer.websocketServer.getConnection(msg.sender);
                    if (!newConnection) {
                        log(2, '%s removes reported session of %s', sessionServer, id);
                        delete sessionServer.trackerData[id];
                        events.add('[connection to %s] remove trackerData', msg.sender);
                    }
                }, sessionServer.inactiveSessionRemovalTime);
            });
        }

        events.add('[sender %s][reportSession]', msg.sender);

        sessionServer.trackerData[id] = {sessions: msg.data[id]};
        connection.send({
            action: msg.action + 'Result',
            inResponseTo: msg.messageId,
            data: {success: true, message: 'Sessions added to ' + sessionServer}
        });
    },

    reportActivity: function(sessionServer, connection, msg) {
        // lively session sends infos about last user activity from time to time
         var sessions = sessionServer.getLocalSessions()[sessionServer.id()],
            session = sessions[msg.sender] = sessions[msg.sender] || {};
        session.lastActivity = msg.data.lastActivity;

        events.add('[sender %s][reportActivity]', msg.sender);

        connection.send({action: msg.action + 'Result', inResponseTo: msg.messageId, data: {success: true}});
    },

    getEventLog: function(sessionServer, connection, msg) {
        var log = events.log.slice()
        if (msg.data.limit) log = log.slice(-1 * msg.data.limit);
        connection.send({
            action: msg.action + 'Result',
            inResponseTo: msg.messageId,
            data: {log: log}
        });
    }
}

var services = require("./LivelyServices").services;
util._extend(services, sessionActions);

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// SessionTracker
// The main server component for managing sessions. A
// session represents a onlinr connection of a lively
// world that can be used to communicate to/from that world
// The SessionTracker will manage the connections to Lively
// worlds and connections to other Lively servers (server-
// to-server connections). Those connections are used to
// a) route messages from one session to another (local
// seends as well as remote sends requiring server-to-server
// routing are supported).
// Also, certain actions (getSessions, reportActivity) are
// directly processed by the tracker and do not require
// connections to other sessions.
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
var websockets = require(path.join(process.env.WORKSPACE_LK, 'core/servers/support/websockets'));
{'fraction_non_alphanumeric': 0.13491983462740748, 'fraction_numerical': 0.004487244126247857, 'mean_word_length': 2.3692882622728044, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3926285', 'n_tokens_mistral': 5433, 'n_tokens_neox': 5166, 'n_words': 1434}