<commit_before>import * as http from "http";
import * as https from "https";
import * as url from "url";
import * as qs from "qs";
import * as fs from "fs";
import * as zlib from "zlib";
import * as path from "path";
import * as formidable from "formidable";
import * as Mime from "mime";
import * as uuid from "uuid";
import * as etag from "etag";
import { BrowserFingerprint } from "browser_fingerprint";
import { api, config, utils, Server, Connection } from "../index";

export class WebServer extends Server {
  server: http.Server | https.Server;
  fingerPrinter: BrowserFingerprint;

  constructor() {
    super();
    this.type = "web";

    this.attributes = {
      canChat: false,
      logConnections: false,
      logExits: false,
      sendWelcomeMessage: false,
      verbs: [], // no verbs for connections of this type, as they are to be very short-lived
    };

    this.connectionCustomMethods = {
      setHeader: (connection: Connection, key, value) => {
        connection.rawConnection.res.setHeader(key, value);
      },

      setStatusCode: (connection: Connection, value) => {
        connection.rawConnection.responseHttpCode = value;
      },

      pipe: (connection: Connection, buffer, headers) => {
        for (const k in headers) {
          //@ts-ignore
          connection.setHeader(k, headers[k]);
        }
        if (typeof buffer === "string") {
          buffer = Buffer.from(buffer);
        }
        connection.rawConnection.res.end(buffer);
      },
    };
  }

  async initialize() {
    if (["api", "file"].indexOf(this.config.rootEndpointType) < 0) {
      throw new Error("rootEndpointType can only be 'api' or 'file'");
    }

    if (
      !this.config.urlPathForFiles &&
      this.config.rootEndpointType === "file"
    ) {
      throw new Error(
        'rootEndpointType cannot be "file" without a urlPathForFiles'
      );
    }

    this.fingerPrinter = new BrowserFingerprint(this.config.fingerprintOptions);
  }

  async start() {
    let bootAttempts = 0;
    if (this.config.secure === false) {
      this.server = http.createServer((req, res) => {
        this.handleRequest(req, res);
      });
    } else {
      this.server = https.createServer(
        this.config.serverOptions,
        (req, res) => {
          this.handleRequest(req, res);
        }
      );
    }

    this.server.on("error", (error) => {
      bootAttempts++;
      if (bootAttempts < this.config.bootAttempts) {
        this.log(`cannot boot web server; trying again [${error}]`, "error");
        if (bootAttempts === 1) {
          this.cleanSocket(this.config.bindIP, this.config.port);
        }
        setTimeout(() => {
          this.log("attempting to boot again..");
          this.server.listen(this.config.port, this.config.bindIP);
        }, 1000);
      } else {
        throw new Error(
          `cannot start web server @ ${this.config.bindIP}:${this.config.port} => ${error}`
        );
      }
    });

    await new Promise((resolve) => {
      this.server.listen(this.config.port, this.config.bindIP, () => {
        this.chmodSocket(this.config.bindIP, this.config.port);
        resolve(null);
      });
    });

    this.on("connection", async (connection: Connection) => {
      const requestMode = await this.determineRequestParams(connection);
      if (requestMode === "api") {
        this.processAction(connection);
      } else if (requestMode === "file") {
        this.processFile(connection);
      } else if (requestMode === "options") {
        this.respondToOptions(connection);
      } else if (requestMode === "trace") {
        this.respondToTrace(connection);
      }
    });

    this.on("actionComplete", this.completeResponse);
  }

  async stop() {
    if (!this.server) return;

    await new Promise((resolve) => {
      this.server.once("close", resolve);
      this.server.close();
    });
  }

  async sendMessage(connection: Connection, message) {<commit_msg>Keep track of the open keep-alive sockets in the web server and close them on shutdown (#1833)<commit_after>import * as http from "http";
import * as https from "https";
import * as url from "url";
import * as qs from "qs";
import * as fs from "fs";
import * as zlib from "zlib";
import * as path from "path";
import * as formidable from "formidable";
import * as Mime from "mime";
import * as uuid from "uuid";
import * as etag from "etag";
import { BrowserFingerprint } from "browser_fingerprint";
import { api, config, utils, Server, Connection } from "../index";

export class WebServer extends Server {
  server: http.Server | https.Server;
  fingerPrinter: BrowserFingerprint;
  sockets: { [id: string]: any };

  constructor() {
    super();
    this.type = "web";
    this.sockets = {};

    this.attributes = {
      canChat: false,
      logConnections: false,
      logExits: false,
      sendWelcomeMessage: false,
      verbs: [], // no verbs for connections of this type, as they are to be very short-lived
    };

    this.connectionCustomMethods = {
      setHeader: (connection: Connection, key, value) => {
        connection.rawConnection.res.setHeader(key, value);
      },

      setStatusCode: (connection: Connection, value) => {
        connection.rawConnection.responseHttpCode = value;
      },

      pipe: (connection: Connection, buffer, headers) => {
        for (const k in headers) {
          //@ts-ignore
          connection.setHeader(k, headers[k]);
        }
        if (typeof buffer === "string") {
          buffer = Buffer.from(buffer);
        }
        connection.rawConnection.res.end(buffer);
      },
    };
  }

  async initialize() {
    if (["api", "file"].indexOf(this.config.rootEndpointType) < 0) {
      throw new Error("rootEndpointType can only be 'api' or 'file'");
    }

    if (
      !this.config.urlPathForFiles &&
      this.config.rootEndpointType === "file"
    ) {
      throw new Error(
        'rootEndpointType cannot be "file" without a urlPathForFiles'
      );
    }

    this.fingerPrinter = new BrowserFingerprint(this.config.fingerprintOptions);
  }

  async start() {
    let bootAttempts = 0;
    if (this.config.secure === false) {
      this.server = http.createServer((req, res) => {
        this.handleRequest(req, res);
      });
    } else {
      this.server = https.createServer(
        this.config.serverOptions,
        (req, res) => {
          this.handleRequest(req, res);
        }
      );
    }

    this.server.on("error", (error) => {
      bootAttempts++;
      if (bootAttempts < this.config.bootAttempts) {
        this.log(`cannot boot web server; trying again [${error}]`, "error");
        if (bootAttempts === 1) {
          this.cleanSocket(this.config.bindIP, this.config.port);
        }
        setTimeout(() => {
          this.log("attempting to boot again..");
          this.server.listen(this.config.port, this.config.bindIP);
        }, 1000);
      } else {
        throw new Error(
          `cannot start web server @ ${this.config.bindIP}:${this.config.port} => ${error}`
        );
      }
    });

    let socketCounter = 0;
    this.server.on("connection", (socket) => {
      const id = socketCounter;
      this.sockets[id] = socket;
      socket.on("close", () => delete this.sockets[id]);
      socketCounter++;
    });

    await new Promise((resolve) => {
      this.server.listen(this.config.port, this.config.bindIP, () => {
        this.chmodSocket(this.config.bindIP, this.config.port);
        resolve(null);
      });
    });

    this.on("connection", async (connection: Connection) => {
      const requestMode = await this.determineRequestParams(connection);
      if (requestMode === "api") {
        this.processAction(connection);
      } else if (requestMode === "file") {
        this.processFile(connection);
      } else if (requestMode === "options") {
        this.respondToOptions(connection);
      } else if (requestMode === "trace") {
        this.respondToTrace(connection);
      }
    });

    this.on("actionComplete", this.completeResponse);
  }

  async stop() {
    if (!this.server) return;

    await new Promise((resolve) => {
      this.server.close(resolve);
      for (const socket of Object.values(this.sockets)) {
        socket.destroy();
      }
    });
  }

  async sendMessage(connection: Connection, message) {
{'fraction_non_alphanumeric': 0.142719961589245, 'fraction_numerical': 0.0022806385788020646, 'mean_word_length': 2.538004246284501, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1503640', 'n_tokens_mistral': 2508, 'n_tokens_neox': 2425, 'n_words': 605}