<commit_before>"use strict";

const Endpoints = require("./Endpoints");
const HTTPS = require("https");
const MultipartData = require("../util/MultipartData");
const SequentialBucket = require("../util/SequentialBucket");
const Zlib = require("zlib");

/**
* Handles API requests
*/
class RequestHandler {
    constructor(client, forceQueueing) {
        this._client = client;
        this.baseURL = Endpoints.BASE_URL;
        this.userAgent = `DiscordBot (https://github.com/abalabahaha/eris, ${require("../../package.json").version})`;
        this.ratelimits = {};
        this.latencyRef = {
            latency: 500,
            raw: [500, 500, 500, 500, 500, 500, 500, 500, 500, 500],
            timeOffset: 0,
            timeOffsets: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
            lastTimeOffsetCheck: 0
        };
        this.globalBlock = false;
        this.readyQueue = [];
        if(forceQueueing) {
            this.globalBlock = true;
            this._client.once("shardPreReady", () => this.globalUnblock());
        }
    }

    globalUnblock() {
        this.globalBlock = false;
        while(this.readyQueue.length > 0) {
            this.readyQueue.shift()();
        }
    }

    routefy(url) {
        var route = url.replace(/\/([a-z-]+)\/(?:[0-9]{17,19})/g, function(match, p) {
            return p === "channels" || p === "guilds" || p === "webhooks" ? match : `/${p}/:id`;
        }).replace(/\/reactions\/[^/]+/g, "/reactions/:id").replace(/^\/webhooks\/(\d+)\/[A-Za-z0-9-_]{64,}/, "/webhooks/$1/:token");
    }

    /**
    * Make an API request
    * @arg {String} method Uppercase HTTP method
    * @arg {String} url URL of the endpoint
    * @arg {Boolean} auth Whether to add the Authorization header and token or not
    * @arg {Object} [body] Request payload
    * @arg {Object} [file] File object
    * @arg {String} file.file A buffer containing file data
    * @arg {String} file.name What to name the file
    * @returns {Promise<Object>} Resolves with the returned JSON data
    */
    request(method, url, auth, body, file, _route, short) {
        var route = _route || this.routefy(url);

        return new Promise((resolve, reject) => {
            var attempts = 0;

            var actualCall = (cb) => {
                var headers = {
                    "User-Agent": this.userAgent,
                    "Accept-Encoding": "gzip,deflate"
                };
                var data;

                try {
                    if(auth) {
                        headers.Authorization = this._client.token;
                    }
                    if(body && body.reason) { // Audit log reason sniping
                        if(body.reason === decodeURI(body.reason)) {
                            body.reason = encodeURIComponent(body.reason);
                        }
                        headers["X-Audit-Log-Reason"] = body.reason;
                        delete body.reason;
                    }
                    if(body && body.queryReason) {
                        body.reason = body.queryReason;
                        delete body.queryReason;
                    }
                    var finalURL = url;
                    if(file) {
                        if(Array.isArray(file)) {
                            data = new MultipartData();
                            headers["Content-Type"] = "multipart/form-data; boundary=" + data.boundary;
                            file.forEach(function(f) {
                                if(!f.file) {
                                    return;
                                }
                                data.attach(f.name, f.file, f.name);
                            });
                            if(body) {
                                data.attach("payload_json", body);
                            }
                            data = data.finish();
                        } else if(file.file) {
                            data = new MultipartData();
                            headers["Content-Type"] = "multipart/form-data; boundary=" + data.boundary;
                            data.attach("file", file.file, file.name);
                            if(body) {
                                data.attach("payload_json", body);
                            }
                            data = data.finish();
                        } else {
                            throw new Error("Invalid file object");
                        }
                    } else if(body) {
                        if(method === "GET" || (method === "PUT" && url.includes("/bans/"))) { // TODO remove PUT case when devs fix
                            var qs = "";
                            Object.keys(body).forEach(function(key) {
                                if(body[key] != undefined) {
                                     if(Array.isArray(body[key])) {
                                        body[key].forEach(function(val) {
                                            qs += `&${encodeURIComponent(key)}=${encodeURIComponent(val)}`;
                                        });
                                    } else {
                                        qs += `&${encodeURIComponent(key)}=${encodeURIComponent(body[key])}`;
                                    }
                                }
                            });
                            finalURL += "?" + qs.substring(1);
                        } else {
                            data = JSON.stringify(body);
                            headers["Content-Type"] = "application/json";
                        }
                    }
                } catch(err) {
                    cb();
                    reject(err);
                    return;
                }

                var req = HTTPS.request({
                    method: method,
                    host: "discordapp.com",
                    path: this.baseURL + finalURL,
                    headers: headers
                });

                var reqError;

                req.once("abort", () => {
                    cb();
                    reqError = reqError || new Error(`Request aborted by client on ${method} ${url}`);
                    reqError.req = req;
                    reject(reqError);
                }).once("aborted", () => {
                    cb();
                    reqError = reqError || new Error(`Request aborted by server on ${method} ${url}`);
                    reqError.req = req;
                    reject(reqError);
                }).once("error", (err) => {
                    reqError = err;
                    req.abort();
                });

                var latency = Date.now();

                req.once("response", (resp) => {
                    latency = Date.now() - latency;
                    this.latencyRef.raw.push(latency);
                    this.latencyRef.latency = ~~(this.latencyRef.latency - this.latencyRef.raw.shift() / 10 + latency / 10);

                    if(this.latencyRef.lastTimeOffsetCheck < Date.now() - 5000) {
                        var timeOffset = ~~((this.latencyRef.lastTimeOffsetCheck = Date.now()) - Date.parse(resp.headers["date"]));
                        if(this.latencyRef.timeOffset - this.latencyRef.latency >= 1000 && timeOffset - this.latencyRef.latency >= 1000) {
                            this._client.emit("error", new Error(`Your clock is ${this.latencyRef.timeOffset}ms behind Discord's server clock. Please check your connection and system time.`));
                        }
                        this.latencyRef.timeOffset = ~~(this.latencyRef.timeOffset - this.latencyRef.timeOffsets.shift() / 10 + timeOffset / 10);
                        console.log(timeOffset, this.latencyRef.timeOffset, Date.parse(resp.headers["date"]));
                        this.latencyRef.timeOffsets.push(timeOffset);
                    }

                    var response = "";

                    var _respStream = resp;
                    if(resp.headers["content-encoding"]) {
                        if(~resp.headers["content-encoding"].indexOf("gzip")) {
                            _respStream = resp.pipe(Zlib.createGunzip());
                        } else if(~resp.headers["content-encoding"].indexOf("deflate")) {
                            _respStream = resp.pipe(Zlib.createInflate());
                        }
                    }

                    _respStream.on("data", (str) => {
                        response += str;
                    }).once("end", () => {
                        var now = Date.now();

                        if(resp.headers["x-ratelimit-limit"]) {
                            this.ratelimits[route].limit = +resp.headers["x-ratelimit-limit"];
                        }

                        this.ratelimits[route].remaining = resp.headers["x-ratelimit-remaining"] === undefined ? 1 : +resp.headers["x-ratelimit-remaining"] || 0;

                        if(resp.headers["retry-after"]) {
                            if(resp.headers["x-ratelimit-global"]) {
                                this.globalBlock = true;
                                setTimeout(() => this.globalUnblock(), +resp.headers["retry-after"] || 1);
                            } else {
                                this.ratelimits[route].reset = (+resp.headers["retry-after"] || 1) + now;
                            }
                        } else if(resp.headers["x-ratelimit-reset"]) {
                            if(route.endsWith("/reactions/:id")) {
                                this.ratelimits[route].reset = now + Math.max(this.latencyRef.timeOffset, 250);
                            } else {
                                this.ratelimits[route].reset = Math.max(+resp.headers["x-ratelimit-reset"] * 1000 + this.latencyRef.timeOffset, now);
                            }
                        } else {
                            this.ratelimits[route].reset = now;
                        }<commit_msg>Warn when unexpectedly missing ratelimit header<commit_after>"use strict";

const Endpoints = require("./Endpoints");
const HTTPS = require("https");
const MultipartData = require("../util/MultipartData");
const SequentialBucket = require("../util/SequentialBucket");
const Zlib = require("zlib");

/**
* Handles API requests
*/
class RequestHandler {
    constructor(client, forceQueueing) {
        this._client = client;
        this.baseURL = Endpoints.BASE_URL;
        this.userAgent = `DiscordBot (https://github.com/abalabahaha/eris, ${require("../../package.json").version})`;
        this.ratelimits = {};
        this.latencyRef = {
            latency: 500,
            raw: [500, 500, 500, 500, 500, 500, 500, 500, 500, 500],
            timeOffset: 0,
            timeOffsets: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
            lastTimeOffsetCheck: 0
        };
        this.globalBlock = false;
        this.readyQueue = [];
        if(forceQueueing) {
            this.globalBlock = true;
            this._client.once("shardPreReady", () => this.globalUnblock());
        }
    }

    globalUnblock() {
        this.globalBlock = false;
        while(this.readyQueue.length > 0) {
            this.readyQueue.shift()();
        }
    }

    routefy(url) {
        var route = url.replace(/\/([a-z-]+)\/(?:[0-9]{17,19})/g, function(match, p) {
            return p === "channels" || p === "guilds" || p === "webhooks" ? match : `/${p}/:id`;
        }).replace(/\/reactions\/[^/]+/g, "/reactions/:id").replace(/^\/webhooks\/(\d+)\/[A-Za-z0-9-_]{64,}/, "/webhooks/$1/:token");
    }

    /**
    * Make an API request
    * @arg {String} method Uppercase HTTP method
    * @arg {String} url URL of the endpoint
    * @arg {Boolean} auth Whether to add the Authorization header and token or not
    * @arg {Object} [body] Request payload
    * @arg {Object} [file] File object
    * @arg {String} file.file A buffer containing file data
    * @arg {String} file.name What to name the file
    * @returns {Promise<Object>} Resolves with the returned JSON data
    */
    request(method, url, auth, body, file, _route, short) {
        var route = _route || this.routefy(url);

        return new Promise((resolve, reject) => {
            var attempts = 0;

            var actualCall = (cb) => {
                var headers = {
                    "User-Agent": this.userAgent,
                    "Accept-Encoding": "gzip,deflate"
                };
                var data;

                try {
                    if(auth) {
                        headers.Authorization = this._client.token;
                    }
                    if(body && body.reason) { // Audit log reason sniping
                        if(body.reason === decodeURI(body.reason)) {
                            body.reason = encodeURIComponent(body.reason);
                        }
                        headers["X-Audit-Log-Reason"] = body.reason;
                        delete body.reason;
                    }
                    if(body && body.queryReason) {
                        body.reason = body.queryReason;
                        delete body.queryReason;
                    }
                    var finalURL = url;
                    if(file) {
                        if(Array.isArray(file)) {
                            data = new MultipartData();
                            headers["Content-Type"] = "multipart/form-data; boundary=" + data.boundary;
                            file.forEach(function(f) {
                                if(!f.file) {
                                    return;
                                }
                                data.attach(f.name, f.file, f.name);
                            });
                            if(body) {
                                data.attach("payload_json", body);
                            }
                            data = data.finish();
                        } else if(file.file) {
                            data = new MultipartData();
                            headers["Content-Type"] = "multipart/form-data; boundary=" + data.boundary;
                            data.attach("file", file.file, file.name);
                            if(body) {
                                data.attach("payload_json", body);
                            }
                            data = data.finish();
                        } else {
                            throw new Error("Invalid file object");
                        }
                    } else if(body) {
                        if(method === "GET" || (method === "PUT" && url.includes("/bans/"))) { // TODO remove PUT case when devs fix
                            var qs = "";
                            Object.keys(body).forEach(function(key) {
                                if(body[key] != undefined) {
                                     if(Array.isArray(body[key])) {
                                        body[key].forEach(function(val) {
                                            qs += `&${encodeURIComponent(key)}=${encodeURIComponent(val)}`;
                                        });
                                    } else {
                                        qs += `&${encodeURIComponent(key)}=${encodeURIComponent(body[key])}`;
                                    }
                                }
                            });
                            finalURL += "?" + qs.substring(1);
                        } else {
                            data = JSON.stringify(body);
                            headers["Content-Type"] = "application/json";
                        }
                    }
                } catch(err) {
                    cb();
                    reject(err);
                    return;
                }

                var req = HTTPS.request({
                    method: method,
                    host: "discordapp.com",
                    path: this.baseURL + finalURL,
                    headers: headers
                });

                var reqError;

                req.once("abort", () => {
                    cb();
                    reqError = reqError || new Error(`Request aborted by client on ${method} ${url}`);
                    reqError.req = req;
                    reject(reqError);
                }).once("aborted", () => {
                    cb();
                    reqError = reqError || new Error(`Request aborted by server on ${method} ${url}`);
                    reqError.req = req;
                    reject(reqError);
                }).once("error", (err) => {
                    reqError = err;
                    req.abort();
                });

                var latency = Date.now();

                req.once("response", (resp) => {
                    latency = Date.now() - latency;
                    this.latencyRef.raw.push(latency);
                    this.latencyRef.latency = ~~(this.latencyRef.latency - this.latencyRef.raw.shift() / 10 + latency / 10);

                    if(this.latencyRef.lastTimeOffsetCheck < Date.now() - 5000) {
                        var timeOffset = ~~((this.latencyRef.lastTimeOffsetCheck = Date.now()) - Date.parse(resp.headers["date"]));
                        if(this.latencyRef.timeOffset - this.latencyRef.latency >= 1000 && timeOffset - this.latencyRef.latency >= 1000) {
                            this._client.emit("error", new Error(`Your clock is ${this.latencyRef.timeOffset}ms behind Discord's server clock. Please check your connection and system time.`));
                        }
                        this.latencyRef.timeOffset = ~~(this.latencyRef.timeOffset - this.latencyRef.timeOffsets.shift() / 10 + timeOffset / 10);
                        console.log(timeOffset, this.latencyRef.timeOffset, Date.parse(resp.headers["date"]));
                        this.latencyRef.timeOffsets.push(timeOffset);
                    }

                    var response = "";

                    var _respStream = resp;
                    if(resp.headers["content-encoding"]) {
                        if(~resp.headers["content-encoding"].indexOf("gzip")) {
                            _respStream = resp.pipe(Zlib.createGunzip());
                        } else if(~resp.headers["content-encoding"].indexOf("deflate")) {
                            _respStream = resp.pipe(Zlib.createInflate());
                        }
                    }

                    _respStream.on("data", (str) => {
                        response += str;
                    }).once("end", () => {
                        var now = Date.now();

                        if(resp.headers["x-ratelimit-limit"]) {
                            this.ratelimits[route].limit = +resp.headers["x-ratelimit-limit"];
                        }

                        if((resp.headers["x-ratelimit-remaining"] == undefined || resp.headers["x-ratelimit-limit"] == undefined) && this.ratelimits[route].limit !== 1) {
                            this._client.emit("warn", "Missing ratelimit headers for SequentialBucket with non-default limit\n"
                                + "x-ratelimit-remaining = " + resp.headers["x-ratelimit-remaining"] + "\n"
                                + "x-ratelimit-limit = " + resp.headers["x-ratelimit-limit"] + "\n"
                                + "x-ratelimit-reset = " + resp.headers["x-ratelimit-reset"] + "\n"
                                + "x-ratelimit-global = " + resp.headers["x-ratelimit-global"]);
                        }

                        this.ratelimits[route].remaining = resp.headers["x-ratelimit-remaining"] === undefined ? 1 : +resp.headers["x-ratelimit-remaining"] || 0;

                        if(resp.headers["retry-after"]) {
                            if(resp.headers["x-ratelimit-global"]) {
                                this.globalBlock = true;
                                setTimeout(() => this.globalUnblock(), +resp.headers["retry-after"] || 1);
                            } else {
                                this.ratelimits[route].reset = (+resp.headers["retry-after"] || 1) + now;
                            }
                        } else if(resp.headers["x-ratelimit-reset"]) {
                            if(route.endsWith("/reactions/:id")) {
                                this.ratelimits[route].reset = now + Math.max(this.latencyRef.timeOffset, 250);
                            } else {
                                this.ratelimits[route].reset = Math.max(+resp.headers["x-ratelimit-reset"] * 1000 + this.latencyRef.timeOffset, now);
                            }
                        } else {
                            this.ratelimits[route].reset = now;
                        }
{'fraction_non_alphanumeric': 0.12943321975332342, 'fraction_numerical': 0.00868647118107213, 'mean_word_length': 1.1337292647962318, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7456328', 'n_tokens_mistral': 5482, 'n_tokens_neox': 5091, 'n_words': 947}