<commit_before>var Promise = require("bluebird");
var bhttp = Promise.promisifyAll(require("bhttp"));
var fs = Promise.promisifyAll(require("fs"));

// config encapsulates opensensors-api-key
// valid keys for config are: api-key (required)
module.exports = function(config) {
    var API_POST_OPTIONS = {
        headers: {
            Accept: "application/json",
            Authorization: "api-key " + config["api-key"]
        }
    };

    var requests_filename = "./current_requests.json";
    var MAX_CONCURRENT_REQUESTS_IN_FLIGHT = 5;

    // on launch delete ther requests file
    if(fs.existsSync(requests_filename)) {
        fs.unlinkSync(requests_filename);
        console.log("Removed existing requests file: " + requests_filename);
    }

    var API_BASE_URL = "https://api.opensensors.io";

    // helper (actually workhorse) method that does a GET to a URL
    // it appends the augmented payloads in the response to the second argument that gets passed to it
    // if the response body JSON contains a next element it recursively calls itself
    var getUntilNot400 = function(url){
        var theUrl = url;
        var requestsInFlight = 0;
        var current_requests = [];
        try{
            var file_contents = fs.readFileSync(requests_filename, 'utf8');
            if(file_contents.trim() == ""){
                file_contents = "[]";
            }
            current_requests = JSON.parse(file_contents);
            requestsInFlight = current_requests.length;
        }
        catch(e){
            console.log(requests_filename + ' is corrupt - JSON parse failed');
        }

        if(requestsInFlight < MAX_CONCURRENT_REQUESTS_IN_FLIGHT) {
            // if the server is not saturated, go ahead and make a request and increase the saturation level

            // add this request to the list of in flight requests
            current_requests.push(theUrl);
            try {
                fs.writeFileSync(requests_filename, JSON.stringify(current_requests));
            }
            catch(e){
                console.log("Failed to write to " + requests_filename);
                // this is very bad... we really shouldn't proceed at this point with this request
                // it should in fact be as though we were saturated
                // just as though we had gotten a 400 response, just try agian sooner
                console.log("Deferring request " + theUrl + " for 5 seconds");
                return Promise.delay(5000).then(function () {
                    return getUntilNot400(theUrl);
                });
            }

            return Promise.try(function () {
                return bhttp.get(theUrl, API_POST_OPTIONS);
            }).then(function (response) {
                if (response.statusCode == 400) {
                    console.log(theUrl);
                    console.log(response.body);
                    console.log("Got 400, waiting 30 seconds before trying " + theUrl + " again");
                    return Promise.delay(30000).then(function () {
                        return getUntilNot400(theUrl);
                    });
                }
                else {
                    // finally! we can retire this request from the list and pass the results on
                    try{
                        var file_contents = fs.readFileSync(requests_filename, 'utf8');
                        if(file_contents.trim() == ""){
                            file_contents = "[]";
                        }
                        current_requests = JSON.parse(file_contents);<commit_msg>don't redundantly add the same url to the request backlog<commit_after>var Promise = require("bluebird");
var bhttp = Promise.promisifyAll(require("bhttp"));
var fs = Promise.promisifyAll(require("fs"));

// config encapsulates opensensors-api-key
// valid keys for config are: api-key (required)
module.exports = function(config) {
    var API_POST_OPTIONS = {
        headers: {
            Accept: "application/json",
            Authorization: "api-key " + config["api-key"]
        }
    };

    var requests_filename = "./current_requests.json";
    var MAX_CONCURRENT_REQUESTS_IN_FLIGHT = 5;

    // on launch delete ther requests file
    if(fs.existsSync(requests_filename)) {
        fs.unlinkSync(requests_filename);
        console.log("Removed existing requests file: " + requests_filename);
    }

    var API_BASE_URL = "https://api.opensensors.io";

    // helper (actually workhorse) method that does a GET to a URL
    // it appends the augmented payloads in the response to the second argument that gets passed to it
    // if the response body JSON contains a next element it recursively calls itself
    var getUntilNot400 = function(url){
        var theUrl = url;
        var requestsInFlight = 0;
        var current_requests = [];
        try{
            var file_contents = fs.readFileSync(requests_filename, 'utf8');
            if(file_contents.trim() == ""){
                file_contents = "[]";
            }
            current_requests = JSON.parse(file_contents);
            requestsInFlight = current_requests.length;
        }
        catch(e){
            console.log(requests_filename + ' is corrupt - JSON parse failed');
        }

        if(requestsInFlight < MAX_CONCURRENT_REQUESTS_IN_FLIGHT) {
            // if the server is not saturated, go ahead and make a request and increase the saturation level

            // add this request to the list of in flight requests
            // if it's not already in the list
            if(current_requests.indexOf(theUrl) == -1){
              current_requests.push(theUrl);
              try {
                  fs.writeFileSync(requests_filename, JSON.stringify(current_requests));
              }
              catch(e){
                  console.log("Failed to write to " + requests_filename);
                  // this is very bad... we really shouldn't proceed at this point with this request
                  // it should in fact be as though we were saturated
                  // just as though we had gotten a 400 response, just try agian sooner
                  console.log("Deferring request " + theUrl + " for 5 seconds");
                  return Promise.delay(5000).then(function () {
                      return getUntilNot400(theUrl);
                  });
              }
            }
            
            return Promise.try(function () {
                return bhttp.get(theUrl, API_POST_OPTIONS);
            }).then(function (response) {
                if (response.statusCode == 400) {
                    console.log(theUrl);
                    console.log(response.body);
                    console.log("Got 400, waiting 30 seconds before trying " + theUrl + " again");
                    return Promise.delay(30000).then(function () {
                        return getUntilNot400(theUrl);
                    });
                }
                else {
                    // finally! we can retire this request from the list and pass the results on
                    try{
                        var file_contents = fs.readFileSync(requests_filename, 'utf8');
                        if(file_contents.trim() == ""){
                            file_contents = "[]";
                        }
                        current_requests = JSON.parse(file_contents);
{'fraction_non_alphanumeric': 0.0918077183480027, 'fraction_numerical': 0.009343263371699391, 'mean_word_length': 1.7683658170914542, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1623806', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1781, 'n_words': 529}