<commit_before>var url = require('url');
var URI = require('URIjs');
var http = require('http');

var ConfigRedis = function (options) {
  this.cwd = '/';
  this.url = options.url;
  this.client = options.client;
};

ConfigRedis.prototype.load = function (hostname, callback) {
  this.getBuild(hostname, function (err, build) {
    if (err) return callback(err);
    
    callback(err, build);
  });
};

ConfigRedis.prototype.getBuild = function (hostname, callback) {
  var self = this;
  this.getBuildId(hostname, function (err, buildId) {
    if (err) return callback(err);
    
    self.client.get(buildId, function (err, buildData) {
      
      // Oops, we don't have the build data
      if (!buildData) {
        return self.constructBuildObject(hostname, function (err, buildId) {
          self.getBuild(hostname, callback);
        });
      }
      
      var build = JSON.parse(buildData);
      build.root = build.config.root || './';
      build.host = hostname;
      build.buildId = buildId;
      build.cwd = buildId;
      
      callback(err, build);
    });
  });
};

ConfigRedis.prototype.getBuildId = function (hostname, callback) {
  var self = this;
  this.client.get(hostname, function (err, buildId) {
    if (!buildId) return self.constructBuildObject(hostname, callback);
    callback(err, buildId);
  });
};

ConfigRedis.prototype.constructBuildObject = function (hostname, callback) {
  var self = this;
  
  // Get info from API server
  this.getRawBuild(hostname, function (err, buildData) {
    if (err) return callback(err);
    
    self.client.set(hostname, buildData.id, function (err, status) {
      
      // Don't save the root, no need for it once it's up on divshot.io
      buildData.config.root = './';
      
      self.client.set(buildData.id, JSON.stringify(buildData), function (err, status) {
        callback(err, buildData.id);
      });
    });
    
  });
};

ConfigRedis.prototype.getRawBuild = function (hostname, callback) {
  http.get(process.env.API_HOST + '/builds/lookup?host=' + hostname, function (res) {
    if (res.statusCode == 404) return callback('NOT_FOUND');
    
    var data = '';
    
    res.on('data', function (chunk) {
      data += chunk.toString();
    }).on('end', function () {
      callback(null, JSON.parse(data));
    });
  });
};<commit_msg>vary net-lib based on protocol<commit_after>var url = require('url');
var URI = require('URIjs');
var http = require('http');
var https = require('https');

var ConfigRedis = function (options) {
  this.cwd = '/';
  this.url = options.url;
  this.client = options.client;
};

ConfigRedis.prototype.load = function (hostname, callback) {
  this.getBuild(hostname, function (err, build) {
    if (err) return callback(err);
    
    callback(err, build);
  });
};

ConfigRedis.prototype.getBuild = function (hostname, callback) {
  var self = this;
  this.getBuildId(hostname, function (err, buildId) {
    if (err) return callback(err);
    
    self.client.get(buildId, function (err, buildData) {
      
      // Oops, we don't have the build data
      if (!buildData) {
        return self.constructBuildObject(hostname, function (err, buildId) {
          self.getBuild(hostname, callback);
        });
      }
      
      var build = JSON.parse(buildData);
      build.root = build.config.root || './';
      build.host = hostname;
      build.buildId = buildId;
      build.cwd = buildId;
      
      callback(err, build);
    });
  });
};

ConfigRedis.prototype.getBuildId = function (hostname, callback) {
  var self = this;
  this.client.get(hostname, function (err, buildId) {
    if (!buildId) return self.constructBuildObject(hostname, callback);
    callback(err, buildId);
  });
};

ConfigRedis.prototype.constructBuildObject = function (hostname, callback) {
  var self = this;
  
  // Get info from API server
  this.getRawBuild(hostname, function (err, buildData) {
    if (err) return callback(err);
    
    self.client.set(hostname, buildData.id, function (err, status) {
      
      // Don't save the root, no need for it once it's up on divshot.io
      buildData.config.root = './';
      
      self.client.set(buildData.id, JSON.stringify(buildData), function (err, status) {
        callback(err, buildData.id);
      });
    });
    
  });
};

ConfigRedis.prototype.getRawBuild = function (hostname, callback) {
  var url = process.env.API_HOST + '/builds/lookup?host=' + hostname;
  var protocol = url.match(/^https/) ? https : http;
  protocol.get(process.env.API_HOST + '/builds/lookup?host=' + hostname, function (res) {
    if (res.statusCode == 404) return callback('NOT_FOUND');
    
    var data = '';
    
    res.on('data', function (chunk) {
      data += chunk.toString();
    }).on('end', function () {
      callback(null, JSON.parse(data));
    });
  });
};
{'fraction_non_alphanumeric': 0.14886128364389234, 'fraction_numerical': 0.0012422360248447205, 'mean_word_length': 2.9436734693877553, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4090726', 'n_tokens_mistral': 1522, 'n_tokens_neox': 1534, 'n_words': 355}