<commit_before>function NetworkError(msg) {
  this.msg = msg;
}
NetworkError.prototype.toString = function() {
  return "Network error";
};
function HttpError(status, statusText, responseText) {
  this.status = status;
  this.statusText = statusText;
  this.responseText = responseText;
}
HttpError.prototype.toString = function() {
  return this.responseText;
};

function corsRequest(method, url, data) {
  var xhr = new XMLHttpRequest();
  if ("withCredentials" in xhr) {
    // XHR with CORS
    xhr.open(method, url, true);
  } else if (typeof XDomainRequest != "undefined") {
    // XDomainRequest for IE9.
    xhr = new XDomainRequest();
    xhr.open(method, url);
  } else {
    return Q.reject("CORS not supported.");
  }

  var deferred = Q.defer();
  xhr.onload = function() {
    if (this.status == 200) {
      deferred.resolve(this.responseText);
    } else if (this.status) {
      deferred.reject(new HttpError(this.status, this.statusText,
                                    this.responseText));
    } else {
      deferred.reject(new NetworkError());
    }
  };
  xhr.onerror = function() {
    deferred.reject("Request failed");
  };

  if (data !== undefined) {
    // TODO(davidben): Make the server handle text/plain Content-Types
    // too for IE9.
    if (xhr.setRequestHeader)
      xhr.setRequestHeader("Content-Type", "application/json");
    xhr.send(JSON.stringify(data));
  } else {
    xhr.send();
  }
  return deferred.promise;
}

var RECONNECT_DELAY = 500;
var RECONNECT_TRIES = 10;

function API(urlBase) {
  io.EventEmitter.call(this);
  this.urlBase_ = urlBase;
  this.token_ = null;

  this.socket_ = null;
  this.socketPending_ = false;
  this.reconnectDelay_ = RECONNECT_DELAY;
  this.reconnectTries_ = RECONNECT_TRIES;
  this.nextTailId_ = 1;

  this.tryConnectSocket_();

  // If we go online, try to reconnect then and there.
  window.addEventListener("online", this.tryConnectSocket_.bind(this));
}
API.prototype = Object.create(io.EventEmitter.prototype);

API.prototype.refreshAuthToken_ = function() {
  // TODO(davidben): Actually authenticate and stuff. This will
  // potentially require emitting an event or something else to pop up
  // a dialog (with a button to Webathena) when cached tickets expire.
  return corsRequest("POST", this.urlBase_ + "/api/v1/auth", {
    principal: "<EMAIL>"
  }).then(function(json) {
    var resp = JSON.parse(json);
    this.token_ = resp.authToken;
    return this.token_;
  }.bind(this));
};

API.prototype.badToken_ = function() {
  console.log("Bad token!");
  this.token_ = null;
};

API.prototype.getAuthToken_ = function() {
  if (this.token_)
    return Q(this.token_);
  return this.refreshAuthToken_();
};

API.prototype.request = function(method, path, params, data, isRetry) {
  return this.getAuthToken_().then(function(token) {
    var url =
      this.urlBase_ + path + "?access_token=" + encodeURIComponent(token);
    for (var key in params) {
      url += "&" + key + "=" + encodeURIComponent(params[key]);
    }
    return corsRequest(method, url, data).then(function(responseText) {
      return JSON.parse(responseText);
    }, function(err) {
      // 401 means we had a bad token (it may have expired). Refresh it.
      if (err instanceof HttpError && err.status == 401) {
        this.badToken_();
        // TODO(davidben): Retry the request after we get a new
        // one. Only retry it once though.
        if (!isRetry)
          return this.request(method, path, params, data, true);
      }
      throw err;
    }.bind(this));
  }.bind(this));
};

API.prototype.get = function(path, params) {
  return this.request("GET", path, params);
};

API.prototype.post = function(path, data) {
  return this.request("POST", path, {}, data);
};

API.prototype.socket = function() {
  return this.socket_;
};

API.prototype.allocateTailId = function() {
  return this.nextTailId_++;
};

API.prototype.tryConnectSocket_ = function() {
  if (this.socket_ || this.socketPending_)
    return;

  this.socketPending_ = true;
  this.getAuthToken_().then(function(token) {
    // Socket.IO's reconnect behavior is weird (it buffers up what you
    // send and such). Don't bother. Also implementing it ourselves
    // means we can integrate with navigator.onLine and the like.
    var url =  this.urlBase_ + "/?access_token=" + encodeURIComponent(token);
    var socket = io.connect(url, {
      "reconnect": false,
      "force new connection": true
    });

    socket.once("connect", function() {
      this.socketPending_ = false;
      this.socket_ = socket;
      // Reset reconnect state.
      this.reconnectDelay_ = RECONNECT_DELAY;
      this.reconnectTries_ = RECONNECT_TRIES;

      this.emit("connect");
      this.socket_.once("disconnect", function() {
        this.emit("disconnect");
        this.socket_ = null;

        setTimeout(this.tryConnectSocket_.bind(this), this.reconnectDelay_);
      }.bind(this));
    }.bind(this));

    // Ensure only one of error and connect_failed are handled.
    var cancelled = false;
    socket.once("error", function(err) {
      if (cancelled) return;
      cancelled = true;
      this.socketPending_ = false;
      // Blegh. Retry with a new token.
      if (err == "handshake unauthorized") {
        this.badToken_();
      }

      // Reconnect with exponential back-off.
      this.reconnectDelay_ *= 2;
      if (this.reconnectTries_-- > 0) {
        setTimeout(this.tryConnectSocket_.bind(this), this.reconnectDelay_);
      }
    }.bind(this));

    socket.once("connect_failed", function() {
      if (cancelled) return;<commit_msg>Better token management with promises<commit_after>function NetworkError(msg) {
  this.msg = msg;
}
NetworkError.prototype.toString = function() {
  return "Network error";
};
function HttpError(status, statusText, responseText) {
  this.status = status;
  this.statusText = statusText;
  this.responseText = responseText;
}
HttpError.prototype.toString = function() {
  return this.responseText;
};

function corsRequest(method, url, data) {
  var xhr = new XMLHttpRequest();
  if ("withCredentials" in xhr) {
    // XHR with CORS
    xhr.open(method, url, true);
  } else if (typeof XDomainRequest != "undefined") {
    // XDomainRequest for IE9.
    xhr = new XDomainRequest();
    xhr.open(method, url);
  } else {
    return Q.reject("CORS not supported.");
  }

  var deferred = Q.defer();
  xhr.onload = function() {
    if (this.status == 200) {
      deferred.resolve(this.responseText);
    } else if (this.status) {
      deferred.reject(new HttpError(this.status, this.statusText,
                                    this.responseText));
    } else {
      deferred.reject(new NetworkError());
    }
  };
  xhr.onerror = function() {
    deferred.reject("Request failed");
  };

  if (data !== undefined) {
    // TODO(davidben): Make the server handle text/plain Content-Types
    // too for IE9.
    if (xhr.setRequestHeader)
      xhr.setRequestHeader("Content-Type", "application/json");
    xhr.send(JSON.stringify(data));
  } else {
    xhr.send();
  }
  return deferred.promise;
}

var RECONNECT_DELAY = 500;
var RECONNECT_TRIES = 10;

function API(urlBase) {
  io.EventEmitter.call(this);
  this.urlBase_ = urlBase;

  this.tokenPromise_ = null;

  this.socket_ = null;
  this.socketPending_ = false;
  this.reconnectDelay_ = RECONNECT_DELAY;
  this.reconnectTries_ = RECONNECT_TRIES;
  this.nextTailId_ = 1;

  this.tryConnectSocket_();

  // If we go online, try to reconnect then and there.
  window.addEventListener("online", this.tryConnectSocket_.bind(this));
}
API.prototype = Object.create(io.EventEmitter.prototype);

API.prototype.refreshAuthToken_ = function() {
  // TODO(davidben): On auth error, reject the ticket and wait for a
  // new one. And on other errors, some sort of exponential back-off I
  // guess.
  //
  // TODO(davidben): Actually authenticate and stuff. This will
  // potentially require emitting an event or something else to pop up
  // a dialog (with a button to Webathena) when cached tickets expire.
  return corsRequest("POST", this.urlBase_ + "/api/v1/auth", {
    principal: "<EMAIL>"
  }).then(function(json) {
    var resp = JSON.parse(json);
    return resp.authToken;
  }.bind(this));
};

API.prototype.badToken_ = function(token) {
  console.log("Bad token!");
  if (this.tokenPromise_ &&
      Q.isFulfilled(this.tokenPromise_) &&
      this.tokenPromise_.valueOf() == token) {
    this.tokenPromise_ = null;
  }
};

API.prototype.getAuthToken_ = function() {
  if (this.tokenPromise_ == null) {
    this.tokenPromise_ = this.refreshAuthToken_();
  }
  return this.tokenPromise_;
};

API.prototype.request = function(method, path, params, data, isRetry) {
  return this.getAuthToken_().then(function(token) {
    var url =
      this.urlBase_ + path + "?access_token=" + encodeURIComponent(token);
    for (var key in params) {
      url += "&" + key + "=" + encodeURIComponent(params[key]);
    }
    return corsRequest(method, url, data).then(function(responseText) {
      return JSON.parse(responseText);
    }, function(err) {
      // 401 means we had a bad token (it may have expired). Refresh it.
      if (err instanceof HttpError && err.status == 401) {
        this.badToken_(token);
        // TODO(davidben): Retry the request after we get a new
        // one. Only retry it once though.
        if (!isRetry)
          return this.request(method, path, params, data, true);
      }
      throw err;
    }.bind(this));
  }.bind(this));
};

API.prototype.get = function(path, params) {
  return this.request("GET", path, params);
};

API.prototype.post = function(path, data) {
  return this.request("POST", path, {}, data);
};

API.prototype.socket = function() {
  return this.socket_;
};

API.prototype.allocateTailId = function() {
  return this.nextTailId_++;
};

API.prototype.tryConnectSocket_ = function() {
  if (this.socket_ || this.socketPending_)
    return;

  this.socketPending_ = true;
  this.getAuthToken_().then(function(token) {
    // Socket.IO's reconnect behavior is weird (it buffers up what you
    // send and such). Don't bother. Also implementing it ourselves
    // means we can integrate with navigator.onLine and the like.
    var url =  this.urlBase_ + "/?access_token=" + encodeURIComponent(token);
    var socket = io.connect(url, {
      "reconnect": false,
      "force new connection": true
    });

    socket.once("connect", function() {
      this.socketPending_ = false;
      this.socket_ = socket;
      // Reset reconnect state.
      this.reconnectDelay_ = RECONNECT_DELAY;
      this.reconnectTries_ = RECONNECT_TRIES;

      this.emit("connect");
      this.socket_.once("disconnect", function() {
        this.emit("disconnect");
        this.socket_ = null;

        setTimeout(this.tryConnectSocket_.bind(this), this.reconnectDelay_);
      }.bind(this));
    }.bind(this));

    // Ensure only one of error and connect_failed are handled.
    var cancelled = false;
    socket.once("error", function(err) {
      if (cancelled) return;
      cancelled = true;
      this.socketPending_ = false;
      // Blegh. Retry with a new token.
      if (err == "handshake unauthorized") {
        this.badToken_(token);
      }

      // Reconnect with exponential back-off.
      this.reconnectDelay_ *= 2;
      if (this.reconnectTries_-- > 0) {
        setTimeout(this.tryConnectSocket_.bind(this), this.reconnectDelay_);
      }
    }.bind(this));

    socket.once("connect_failed", function() {
      if (cancelled) return;
{'fraction_non_alphanumeric': 0.13477018956115294, 'fraction_numerical': 0.0034623041634207563, 'mean_word_length': 3.09861653068464, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1247895', 'n_tokens_mistral': 3671, 'n_tokens_neox': 3640, 'n_words': 877}