<commit_before>var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest,
    async = require("async"),
    Class = require("./inheritance").Class,
    crypto = require('crypto'),
    error = require("./error"),
    urljoin = require("./utils").urljoin,
    is_ans1_token = require("./utils").is_ans1_token;


var Client = Class.extend({

  // Base client version
  VERSION: "1.0",
  redacted_request: ['password'],
  redacted_response: [
    'private_key',
    // TODO: This is a mega hack to redact the private key in certain situation
    //       problem is that properties are ordered non-deterministically so "data"
    //       could come before "private_key" we need a better way to do this. But it
    //       works for now.
    'private_key": null, "data'
  ],

  init: function (options) {
    options = options || {};
    this.debug = options.debug || false;
    this.log_level = this.debug ? "debug" : (options.log_level || "warning");
    this._log_level = this.log_levels[this.log_level];  // Store the numeric version so we don't recalculate it every time.
    this.truncate_long_response = options.truncate_long_response || true; // Set default truncation to truncate...
    this.truncate_response_at = options.truncate_response_at || -1; // but only based on specific truncation lengths in params.
    this.url = options.url;
    this.scoped_token = options.scoped_token || null;
    this.unscoped_token = options.unscoped_token || null;
    this.service_catalog = options.service_catalog || [];
    this.tenant = options.project || null;
    this.user = options.user || null;
  },

  log_levels: {
    "critical": 100,
    "error": 80,
    "warn": 60,
    "info": 40,
    "debug": 20
  },

  // Generic logging placeholder.
  // TODO(gabriel): Make this better.
  log: function (level) {
    if (typeof level !== "number") level = this.log_levels[level];
    if (level >= this._log_level) {
      console.log(Array.prototype.slice.apply(arguments, [1, arguments.length]).join(" "));
    }
  },

  redact: function (json_string, redacted) {
    for (var i = 0; i < redacted.length; i++) {
      var re = new RegExp('("' + redacted[i] + '":\\s?)"(([^\\"]|\\\\|\\")*)"', "g");
      json_string = json_string.replace(re, '$1"*****"');
    }
    return json_string;
  },

  // Format headers to pretty-print for easier reading.
  format_headers: function (headers) {
    var formatted = "";
    for (var header in headers) {
      if (headers.hasOwnProperty(header)) {
        formatted += "\n" + header + ": " + headers[header];
      }
    }
    return formatted;
  },

  // Fetches a URL for the current service type with the given endpoint type.
  url_for: function (endpoint_type, service_type) {
    var search_service_type = service_type || this.service_type;
    for (var i = 0; i < this.service_catalog.length; i++) {
      if (this.service_catalog[i].type === search_service_type) {
        return this.service_catalog[i].endpoints[0][endpoint_type];
      }
    }
    if (service_type) {  // If we came up empty for a specific search, return null.
      return null;
    } else {  // Otherwise try returning a pre-set URL.
      return this.url;
    }
  },

  // Core method for making requests to API endpoints.
  // All other methods eventually route back to this one.
  request: function (params, callback) {
    var xhr = new XMLHttpRequest(),
        client = this,
        token = this.scoped_token || this.unscoped_token,
        url = params.url,
        dataType,
        data,
        result,
        headers,
        method;

    // This is mainly necessary due to Glance needing the Content-Length
    // header set on PUT requests, but xmlhttprequest only setting it for POST.
    if (params.allow_headers) xhr.setDisableHeaderCheck(true);

    if (params.query) {
      var query_params = [];
      Object.keys(params.query).forEach(function (key) {
        query_params.push(key + "=" + params.query[key]);
      });
      url += "?" + query_params.join("&");
    }

    xhr.open(params.method, url, true);

    method = params.method.toUpperCase();

    headers = params.headers || {};
    if (!headers["Content-Type"]) headers["Content-Type"] = "application/json";
    headers.Accept = "application/json";

    // Set our auth token if we have one.
    if (token) {
      headers['X-Auth-Token'] = token.id;
    }

    // Create our XMLHttpRequest and set headers.
    for (var header in headers) {
      if (headers.hasOwnProperty(header)) {
        xhr.setRequestHeader(header, headers[header]);
      }
    }

    function log_request(level, method, url, headers, data) {
      var args = [level, "\nREQ:", method, url, client.format_headers(headers)];
      if (data) args = args.concat(["\nbody:", client.redact(data, client.redacted_request)]);
      client.log.apply(client, args);
    }

    function log_response(level, method, url, status, headers, data) {
      var args = [level, "\nRES:", method, url, "\nstatus:", status, "\n", headers];
      if (data) args = args.concat(["\nbody:", client.redact(data, client.redacted_response)]);
      client.log.apply(client, args);
    }

    function end(err) {
      if (err && params.error) {
        params.error(err, xhr);
      }
      if (!err && params.success) {
        params.success(result, xhr);
      }
      if (params.complete) {
        params.complete(result, xhr, err);
      }
      if (callback) {
        callback(err, result, xhr);
      }
    }

    // Set up our state change handlers.
    xhr.onreadystatechange = function () {
      var status = parseInt(xhr.status, 10);<commit_msg>Allows a defer callback for a request<commit_after>var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest,
    async = require("async"),
    Class = require("./inheritance").Class,
    crypto = require('crypto'),
    error = require("./error"),
    urljoin = require("./utils").urljoin,
    is_ans1_token = require("./utils").is_ans1_token;


var Client = Class.extend({

  // Base client version
  VERSION: "1.0",
  redacted_request: ['password'],
  redacted_response: [
    'private_key',
    // TODO: This is a mega hack to redact the private key in certain situation
    //       problem is that properties are ordered non-deterministically so "data"
    //       could come before "private_key" we need a better way to do this. But it
    //       works for now.
    'private_key": null, "data'
  ],

  init: function (options) {
    options = options || {};
    this.debug = options.debug || false;
    this.log_level = this.debug ? "debug" : (options.log_level || "warning");
    this._log_level = this.log_levels[this.log_level];  // Store the numeric version so we don't recalculate it every time.
    this.truncate_long_response = options.truncate_long_response || true; // Set default truncation to truncate...
    this.truncate_response_at = options.truncate_response_at || -1; // but only based on specific truncation lengths in params.
    this.url = options.url;
    this.scoped_token = options.scoped_token || null;
    this.unscoped_token = options.unscoped_token || null;
    this.service_catalog = options.service_catalog || [];
    this.tenant = options.project || null;
    this.user = options.user || null;
  },

  log_levels: {
    "critical": 100,
    "error": 80,
    "warn": 60,
    "info": 40,
    "debug": 20
  },

  // Generic logging placeholder.
  // TODO(gabriel): Make this better.
  log: function (level) {
    if (typeof level !== "number") level = this.log_levels[level];
    if (level >= this._log_level) {
      console.log(Array.prototype.slice.apply(arguments, [1, arguments.length]).join(" "));
    }
  },

  redact: function (json_string, redacted) {
    for (var i = 0; i < redacted.length; i++) {
      var re = new RegExp('("' + redacted[i] + '":\\s?)"(([^\\"]|\\\\|\\")*)"', "g");
      json_string = json_string.replace(re, '$1"*****"');
    }
    return json_string;
  },

  // Format headers to pretty-print for easier reading.
  format_headers: function (headers) {
    var formatted = "";
    for (var header in headers) {
      if (headers.hasOwnProperty(header)) {
        formatted += "\n" + header + ": " + headers[header];
      }
    }
    return formatted;
  },

  // Fetches a URL for the current service type with the given endpoint type.
  url_for: function (endpoint_type, service_type) {
    var search_service_type = service_type || this.service_type;
    for (var i = 0; i < this.service_catalog.length; i++) {
      if (this.service_catalog[i].type === search_service_type) {
        return this.service_catalog[i].endpoints[0][endpoint_type];
      }
    }
    if (service_type) {  // If we came up empty for a specific search, return null.
      return null;
    } else {  // Otherwise try returning a pre-set URL.
      return this.url;
    }
  },

  // Core method for making requests to API endpoints.
  // All other methods eventually route back to this one.
  request: function (params, callback) {
    var xhr = new XMLHttpRequest(),
        client = this,
        token = this.scoped_token || this.unscoped_token,
        url = params.url,
        dataType,
        data,
        result,
        headers,
        method;

    // This is mainly necessary due to Glance needing the Content-Length
    // header set on PUT requests, but xmlhttprequest only setting it for POST.
    if (params.allow_headers) xhr.setDisableHeaderCheck(true);

    if (params.query) {
      var query_params = [];
      Object.keys(params.query).forEach(function (key) {
        query_params.push(key + "=" + params.query[key]);
      });
      url += "?" + query_params.join("&");
    }

    xhr.open(params.method, url, true);

    method = params.method.toUpperCase();

    headers = params.headers || {};
    if (!headers["Content-Type"]) headers["Content-Type"] = "application/json";
    headers.Accept = "application/json";

    // Set our auth token if we have one.
    if (token) {
      headers['X-Auth-Token'] = token.id;
    }

    // Create our XMLHttpRequest and set headers.
    for (var header in headers) {
      if (headers.hasOwnProperty(header)) {
        xhr.setRequestHeader(header, headers[header]);
      }
    }

    function log_request(level, method, url, headers, data) {
      var args = [level, "\nREQ:", method, url, client.format_headers(headers)];
      if (data) args = args.concat(["\nbody:", client.redact(data, client.redacted_request)]);
      client.log.apply(client, args);
    }

    function log_response(level, method, url, status, headers, data) {
      var args = [level, "\nRES:", method, url, "\nstatus:", status, "\n", headers];
      if (data) args = args.concat(["\nbody:", client.redact(data, client.redacted_response)]);
      client.log.apply(client, args);
    }

    function end(err) {
      if (!err && params.defer) {
        return params.defer(result, function (e, r) {
          params.defer = null;
          result = r || result;
          end(e);
        });
      }

      if (err && params.error) {
        params.error(err, xhr);
      }
      if (!err && params.success) {
        params.success(result, xhr);
      }
      if (params.complete) {
        params.complete(result, xhr, err);
      }
      if (callback) {
        callback(err, result, xhr);
      }
    }

    // Set up our state change handlers.
    xhr.onreadystatechange = function () {
      var status = parseInt(xhr.status, 10);
{'fraction_non_alphanumeric': 0.14285714285714285, 'fraction_numerical': 0.004021682112257388, 'mean_word_length': 2.715167262098084, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6385835', 'n_tokens_mistral': 3601, 'n_tokens_neox': 3470, 'n_words': 943}