<commit_before>var ws = require("ws");
var request = require("request");

var WebSocketServer = ws.Server;
var wss = new WebSocketServer({port: 5000});

var unsentClientMessages = {};
var newMessageURLs = {}; //POST requests not acknowledged by server

var getUniqueId = function() {
    var i = 0;
    var plusOne = function() {
        i += 1;
        return i;
    };
    return plusOne;
}();

wss.on("connection", function(client) {

    client.toString = function() {

        //returns cookie for the hashing objects in unsentClientMessages. 
        /*
        client.upgradeReq.headers.cookie
        "sessionid=lfmfdl637gdrawvhdyhxk9suydcjte7d; csrftoken=<KEY>"
        */
        if(!client._uniqueId) {
            client._uniqueId = getUniqueId();
        }
        return client.upgradeReq.headers.cookie + "-" + client._uniqueId;
    };

    console.log("Client Connected", client.toString());
    unsentClientMessages[client] = {};

    client.on("message", function(stringMessage) {
        
        console.log("Received message");
        var message = JSON.parse(stringMessage);

        if(message.httpMethod === 'POST') {
            newMessageURLs[message.url] = true;
            saveMessage(client, message);
            return;
        }


        var isSaveMessageScheduled = !(!unsentClientMessages[client][message.url]);
        unsentClientMessages[client][message.url] = message;   
        
        if(!isSaveMessageScheduled) {
            console.log("Scheduling save message");
            scheduleSaveMessage(client, message);    
        } 
        else {
            console.log("Message scheduled already. NOT scheduling save message.");
        }

    });

    client.on("close", function() {
        cleanupClientMessages(client);
        console.log("Client closed", client.toString());
    });
});


var lastScheduledSaveTime = null;
var MAX_PUT_REQUESTS_PER_SECOND = 10;
var MINIMUM_REQUEST_GAP =  1000 / MAX_PUT_REQUESTS_PER_SECOND;

scheduleSaveMessage = function(client, message) {

    var waitFor;
    var now = new Date();
    if(lastScheduledSaveTime === null || (now - lastScheduledSaveTime) > MINIMUM_REQUEST_GAP) {
        waitFor = 0;
        lastScheduledSaveTime = now;
    }
    else {
        var scheduledSaveTime = new Date(lastScheduledSaveTime + MINIMUM_REQUEST_GAP); 
        waitFor = (now - scheduledSaveTime);
        lastScheduledSaveTime = scheduledSaveTime;

    }
    console.log("Next PUT request after ", waitFor, "ms");
    saveMessageWithTimeout(client, message, waitFor);
}


var saveMessageWithTimeout = function(client, message, waitFor) {
    
    setTimeout(function() {
        if(newMessageURLs[message.url]) {
            //Response not received for POST yet. 
            console.log("POST response NOT received for ", message.url, "Rescheduling request");
            scheduleSaveMessage(client, message);
        }
        else {
            //save latest message stored on buffer
            saveMessage(client, unsentClientMessages[client][message.url]);    
        }
        
    }, waitFor);
};

var BASE_URL = "http://localhost:7777";

var saveMessage = function(client, message) {
    var method;
    //console.log("message ", message);

    var url = BASE_URL + message.url;
    var httpMethod = message.httpMethod.toLowerCase();
    
    var requestMethod = request[httpMethod];
    var headers = client.upgradeReq.headers;

    headers['Accept'] = 'application/json';
    headers['Content-Type'] = 'application/json';
    headers['X-CSRFToken'] = message.csrfToken;

    var options = {
        url: url,
        headers: headers,
        body: JSON.stringify(message.payload)
    }

    //to minimize memory occupied by closure
    _saveMessageImpl(client, options, requestMethod, message.url);

    console.log("Saving message and deleting it from buffer", message.url);
    delete unsentClientMessages[client][message.url];
};

var _saveMessageImpl = function(client, options, requestMethod, url) {

    var callback = function(error, response, body) {
        if(response.statusCode === 200 || response.statusCode === 201) {
            console.log("Saved message: ", url);
        }
        else {
            console.warn("Unable to save message: ", response.statusCode, response.body);
        }
        if(newMessageURLs[url]) {
            delete newMessageURLs[url];
            console.log("POST response received for", url, "Ready to send PUT requests");
        }
        var message = {
            payload: response.body,
            statusCode: response.statusCode,
            url: url
        };

        try {
            client.send(JSON.stringify(message));
        } catch(e) {
            console.log("Unable to send response. Client ", client.toString(), "has closed the connection", e.toString())
        }
    }

    requestMethod(options, callback);
}

<commit_msg>WebSocket: Added absolute URL when logging saved messages.<commit_after>var ws = require("ws");
var request = require("request");

var WebSocketServer = ws.Server;
var wss = new WebSocketServer({port: 5000});

var unsentClientMessages = {};
var newMessageURLs = {}; //POST requests not acknowledged by server

var getUniqueId = function() {
    var i = 0;
    var plusOne = function() {
        i += 1;
        return i;
    };
    return plusOne;
}();

wss.on("connection", function(client) {

    client.toString = function() {

        //returns cookie for the hashing objects in unsentClientMessages. 
        /*
        client.upgradeReq.headers.cookie
        "sessionid=lfmfdl637gdrawvhdyhxk9suydcjte7d; csrftoken=<KEY>"
        */
        if(!client._uniqueId) {
            client._uniqueId = getUniqueId();
        }
        return client.upgradeReq.headers.cookie + "-" + client._uniqueId;
    };

    console.log("Client Connected", client.toString());
    unsentClientMessages[client] = {};

    client.on("message", function(stringMessage) {
        
        console.log("Received message");
        var message = JSON.parse(stringMessage);

        if(message.httpMethod === 'POST') {
            newMessageURLs[message.url] = true;
            saveMessage(client, message);
            return;
        }


        var isSaveMessageScheduled = !(!unsentClientMessages[client][message.url]);
        unsentClientMessages[client][message.url] = message;   
        
        if(!isSaveMessageScheduled) {
            console.log("Scheduling save message");
            scheduleSaveMessage(client, message);    
        } 
        else {
            console.log("Message scheduled already. NOT scheduling save message.");
        }

    });

    client.on("close", function() {
        cleanupClientMessages(client);
        console.log("Client closed", client.toString());
    });
});


var lastScheduledSaveTime = null;
var MAX_PUT_REQUESTS_PER_SECOND = 10;
var MINIMUM_REQUEST_GAP =  1000 / MAX_PUT_REQUESTS_PER_SECOND;

scheduleSaveMessage = function(client, message) {

    var waitFor;
    var now = new Date();
    if(lastScheduledSaveTime === null || (now - lastScheduledSaveTime) > MINIMUM_REQUEST_GAP) {
        waitFor = 0;
        lastScheduledSaveTime = now;
    }
    else {
        var scheduledSaveTime = new Date(lastScheduledSaveTime + MINIMUM_REQUEST_GAP); 
        waitFor = (now - scheduledSaveTime);
        lastScheduledSaveTime = scheduledSaveTime;

    }
    console.log("Next PUT request after ", waitFor, "ms");
    saveMessageWithTimeout(client, message, waitFor);
}


var saveMessageWithTimeout = function(client, message, waitFor) {
    
    setTimeout(function() {
        if(newMessageURLs[message.url]) {
            //Response not received for POST yet. 
            console.log("POST response NOT received for ", message.url, "Rescheduling request");
            scheduleSaveMessage(client, message);
        }
        else {
            //save latest message stored on buffer
            saveMessage(client, unsentClientMessages[client][message.url]);    
        }
        
    }, waitFor);
};

var BASE_URL = "http://localhost:7777";

var saveMessage = function(client, message) {
    var method;
    //console.log("message ", message);

    var url = BASE_URL + message.url;
    var httpMethod = message.httpMethod.toLowerCase();
    
    var requestMethod = request[httpMethod];
    var headers = client.upgradeReq.headers;

    headers['Accept'] = 'application/json';
    headers['Content-Type'] = 'application/json';
    headers['X-CSRFToken'] = message.csrfToken;

    var options = {
        url: url,
        headers: headers,
        body: JSON.stringify(message.payload)
    }

    //to minimize memory occupied by closure
    _saveMessageImpl(client, options, requestMethod, message.url);

    console.log("Saving message and deleting it from buffer", message.url);
    delete unsentClientMessages[client][message.url];
};

var _saveMessageImpl = function(client, options, requestMethod, url) {

    var callback = function(error, response, body) {
        if(response.statusCode === 200 || response.statusCode === 201) {
            console.log("Saved message: ", options.url);
        }
        else {
            console.warn("Unable to save message: ", response.statusCode, response.body);
        }
        if(newMessageURLs[url]) {
            delete newMessageURLs[url];
            console.log("POST response received for", url, "Ready to send PUT requests");
        }
        var message = {
            payload: response.body,
            statusCode: response.statusCode,
            url: url
        };

        try {
            client.send(JSON.stringify(message));
        } catch(e) {
            console.log("Unable to send response. Client ", client.toString(), "has closed the connection", e.toString())
        }
    }

    requestMethod(options, callback);
}


{'fraction_non_alphanumeric': 0.11712628472575558, 'fraction_numerical': 0.005698585529663173, 'mean_word_length': 2.576419213973799, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2196544', 'n_tokens_mistral': 2797, 'n_tokens_neox': 2690, 'n_words': 615}