<commit_before>const fs = require("fs");
const gzip = require("./gzip");
const http = require("http");
const ipInfo = require("./ipInfo");
const path = require("path");
const requestIp = require("request-ip");
const url = require("url");

const Logger = require("./logger");
const WebSocketServer = require("./wsServer");
const DownloadData = require("./downloadData");
const ResultWritter = require("./resultWritter");

const pagesUriList = [
  "/",
  "/about",
  "/result",
  "/results",
  "/run",
  "/share",
  "/settings"
];

class HttpServer {
  constructor(port, webPath) {
    this.serverFolderPath = process.cwd();
    this.webFolderPath = path.isAbsolute(webPath)
      ? webPath
      : path.join(this.serverFolderPath, webPath);
    this.logger = new Logger();
    this.downloadData = new DownloadData();
    this.resultWritter = new ResultWritter(this.serverFolderPath);

    if (!fs.existsSync(this.webFolderPath)) {
      this.logger.error(
        `The web folder does not exists: ${this.webFolderPath}`
      );
      return;
    }

    this.httpServer = this.createServer(port);
    if (!this.httpServer.listening) {
      throw new Error(`Server failed listening on port ${port}`);
    }
    this.logger.log(`HTTP server listening at http://0.0.0.0:${port}/`);

    this.wsServer = new WebSocketServer(this.httpServer);
    this.logger.log(`WebSocket server listening at ws://0.0.0.0:${port}/`);

    return this.httpServer;
  }

  createServer(port) {
    return http
      .createServer((request, response) => {
        this.logger.debug(`Received request for ${request.url}`);
        const { pathname: uri, query: query } = url.parse(request.url, true);

        if (uri === "/ip") return this.writeIpInfo(request, response);
        if (uri === "/ping" || uri === "/upload")
          return this.writeEmpty(response);
        if (uri === "/download") return this.writeDownloadData(query, response);
        if (uri === "/save" && request.method === "POST")
          return this.saveResult(request, response);
        if (uri.startsWith("/results/")) return this.getResults(uri, response);

        this.loadFile(uri, request, response);
      })
      .listen(port);
  }

  writeIpInfo(request, response) {
    const ipRegex = /\:\:ffff\:((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/;
    const ip = ipRegex.test(requestIp.getClientIp(request))
      ? request.connection.remoteAddress.replace("::ffff:", "")
      : request.connection.remoteAddress;

    let info;
    ipInfo(ip)
      .then(response => (info = response))
      .catch(() => (info = { ip }))
      .finally(() => {
        info = Buffer.from(JSON.stringify(info));
        response.writeHead(200, {
          "Content-Type": "application/json",
          "Content-Length": info.byteLength
        });
        response.write(info);
        response.end();
      });
  }

  writeEmpty(response) {
    response.writeHead(200);
    response.end();
  }

  writeDownloadData(query, response) {
    const size = query.size || 8 * 1024 ** 2;
    response.writeHead(200, {
      "Content-Type": "application/octet-stream",
      "Content-Length": size
    });
    for (let chunk of this.downloadData.get(size)) {
      response.write(chunk, "binary");
    }
    response.end();
  }

  saveResult(request, response) {
    let data = [];
    request.on("data", chunk => data.push(chunk));
    request.on("end", () => {
      this.resultWritter
        .store(JSON.parse(Buffer.concat(data).toString()))
        .then(id => {
          response.writeHead(200, {
            "Content-Type": "text/plain",
            "Content-Length": id.length
          });
          response.write(id);
          response.end();
        })
        .catch(reason => {
          response.writeHead(500, {
            "Content-Type": "text/plain"
          });
          response.write(reason.name);
          response.end();<commit_msg>allow cross origin requests<commit_after>const fs = require("fs");
const gzip = require("./gzip");
const http = require("http");
const ipInfo = require("./ipInfo");
const path = require("path");
const requestIp = require("request-ip");
const url = require("url");

const Logger = require("./logger");
const WebSocketServer = require("./wsServer");
const DownloadData = require("./downloadData");
const ResultWritter = require("./resultWritter");

const pagesUriList = [
  "/",
  "/about",
  "/result",
  "/results",
  "/run",
  "/share",
  "/settings"
];

class HttpServer {
  constructor(port, webPath) {
    this.serverFolderPath = process.cwd();
    this.webFolderPath = path.isAbsolute(webPath)
      ? webPath
      : path.join(this.serverFolderPath, webPath);
    this.logger = new Logger();
    this.downloadData = new DownloadData();
    this.resultWritter = new ResultWritter(this.serverFolderPath);

    if (!fs.existsSync(this.webFolderPath)) {
      this.logger.error(
        `The web folder does not exists: ${this.webFolderPath}`
      );
      return;
    }

    this.httpServer = this.createServer(port);
    if (!this.httpServer.listening) {
      throw new Error(`Server failed listening on port ${port}`);
    }
    this.logger.log(`HTTP server listening at http://0.0.0.0:${port}/`);

    this.wsServer = new WebSocketServer(this.httpServer);
    this.logger.log(`WebSocket server listening at ws://0.0.0.0:${port}/`);

    return this.httpServer;
  }

  createServer(port) {
    return http
      .createServer((request, response) => {
        this.logger.debug(`Received request for ${request.url}`);
        const { pathname: uri, query: query } = url.parse(request.url, true);

        if (uri === "/ip") return this.writeIpInfo(request, response);
        if (uri === "/ping" || uri === "/upload")
          return this.writeEmpty(response);
        if (uri === "/download") return this.writeDownloadData(query, response);
        if (uri === "/save" && request.method === "POST")
          return this.saveResult(request, response);
        if (uri.startsWith("/results/")) return this.getResults(uri, response);

        this.loadFile(uri, request, response);
      })
      .listen(port);
  }

  writeIpInfo(request, response) {
    const ipRegex = /\:\:ffff\:((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/;
    const ip = ipRegex.test(requestIp.getClientIp(request))
      ? request.connection.remoteAddress.replace("::ffff:", "")
      : request.connection.remoteAddress;

    let info;
    ipInfo(ip)
      .then(response => (info = response))
      .catch(() => (info = { ip }))
      .finally(() => {
        info = Buffer.from(JSON.stringify(info));
        response.writeHead(200, {
          "Access-Control-Allow-Origin": "*",
          "Content-Type": "application/json",
          "Content-Length": info.byteLength
        });
        response.write(info);
        response.end();
      });
  }

  writeEmpty(response) {
    response.writeHead(200, {
      "Access-Control-Allow-Origin": "*"
    });
    response.end();
  }

  writeDownloadData(query, response) {
    const size = query.size || 8 * 1024 ** 2;
    response.writeHead(200, {
      "Access-Control-Allow-Origin": "*",
      "Content-Type": "application/octet-stream",
      "Content-Length": size
    });
    for (let chunk of this.downloadData.get(size)) {
      response.write(chunk, "binary");
    }
    response.end();
  }

  saveResult(request, response) {
    let data = [];
    request.on("data", chunk => data.push(chunk));
    request.on("end", () => {
      this.resultWritter
        .store(JSON.parse(Buffer.concat(data).toString()))
        .then(id => {
          response.writeHead(200, {
            "Access-Control-Allow-Origin": "*",
            "Content-Type": "text/plain",
            "Content-Length": id.length
          });
          response.write(id);
          response.end();
        })
        .catch(reason => {
          response.writeHead(500, {
            "Content-Type": "text/plain"
          });
          response.write(reason.name);
          response.end();
{'fraction_non_alphanumeric': 0.17850953206239167, 'fraction_numerical': 0.014855162168853677, 'mean_word_length': 2.7059633027522936, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2743034', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2493, 'n_words': 462}