<commit_before>#!/usr/bin/env node

"use strict";

require("babel-register");

const path = require("path");
const webpack = require("webpack");
const express = require("express");
const webpackDevMiddleware = require("webpack-dev-middleware");
const webpackHotMiddleware = require("webpack-hot-middleware");
const http = require("http");

// Setup Config
const getConfig = require("../config/config").getConfig;
const feature = require("../config/feature");
const config = getConfig();
feature.setConfig(config);

if (!feature.isEnabled("firefox.webSocketConnection")) {
  const firefoxProxy = require("./firefox-proxy");
  firefoxProxy({ logging: feature.isEnabled("logging.firefoxProxy") });
}

function httpGet(url, onResponse) {
  return http.get(url, (response) => {
    let body = "";
    response.on("data", function(d) {
      body += d;
    });
    response.on("end", () => onResponse(body));
  });
}

const app = express();

// Webpack middleware
const webpackConfig = require("../webpack.config");
const compiler = webpack(webpackConfig);

app.use(webpackDevMiddleware(compiler, {
  publicPath: webpackConfig.output.publicPath,
  noInfo: true,
  stats: { colors: true }
}));

if (feature.isEnabled("hotReloading")) {
  app.use(webpackHotMiddleware(compiler));
} else {
  console.log("Hot Reloading can be enabled by adding " +
  "\"hotReloading\": true to your local.json config");
}

// Static middleware
app.use(express.static("public"));

// Routes
app.get("/", function(req, res) {
  res.sendFile(path.join(__dirname, "../index.html"));
});

app.get("/get", function(req, res) {
  const httpReq = httpGet(
    req.query.url,
    body => res.json(JSON.parse(body))
  );

  httpReq.on("error", err => res.status(500).send(err.code));
});

// Listen
app.listen(8000, "0.0.0.0", function(err, result) {
  if (err) {
    console.log(err);
  }

  console.log("Development Server Listening at http://localhost:8000");
});

const examples = express();
examples.use(express.static("public/js/test/examples"));
examples.listen(7999, "0.0.0.0", function(err, result) {
  console.log("View debugger examples at http://localhost:7999");
});<commit_msg>prevent http get crash (#389)<commit_after>#!/usr/bin/env node

"use strict";

require("babel-register");

const path = require("path");
const webpack = require("webpack");
const express = require("express");
const webpackDevMiddleware = require("webpack-dev-middleware");
const webpackHotMiddleware = require("webpack-hot-middleware");
const http = require("http");

// Setup Config
const getConfig = require("../config/config").getConfig;
const feature = require("../config/feature");
const config = getConfig();
feature.setConfig(config);

if (!feature.isEnabled("firefox.webSocketConnection")) {
  const firefoxProxy = require("./firefox-proxy");
  firefoxProxy({ logging: feature.isEnabled("logging.firefoxProxy") });
}

function httpGet(url, onResponse) {
  return http.get(url, (response) => {
    if (response.statusCode !== 200) {
      console.error(`error response: ${response.statusCode} to ${url}`);
      response.emit("statusCode", new Error(response.statusCode));
      return onResponse("{}");
    }
    let body = "";
    response.on("data", function(d) {
      body += d;
    });
    response.on("end", () => onResponse(body));
  });
}

const app = express();

// Webpack middleware
const webpackConfig = require("../webpack.config");
const compiler = webpack(webpackConfig);

app.use(webpackDevMiddleware(compiler, {
  publicPath: webpackConfig.output.publicPath,
  noInfo: true,
  stats: { colors: true }
}));

if (feature.isEnabled("hotReloading")) {
  app.use(webpackHotMiddleware(compiler));
} else {
  console.log("Hot Reloading can be enabled by adding " +
  "\"hotReloading\": true to your local.json config");
}

// Static middleware
app.use(express.static("public"));

// Routes
app.get("/", function(req, res) {
  res.sendFile(path.join(__dirname, "../index.html"));
});

app.get("/get", function(req, res) {
  const httpReq = httpGet(
    req.query.url,
    body => {
      try {
        return res.json(JSON.parse(body));
      } catch (e) {
        throw Error(e);
      }
    }
  );

  httpReq.on("error", err => res.status(500).send(err.code));
  httpReq.on("statusCode", err => res.status(err.message).send(err.message));
});

// Listen
app.listen(8000, "0.0.0.0", function(err, result) {
  if (err) {
    console.log(err);
  }

  console.log("Development Server Listening at http://localhost:8000");
});

const examples = express();
examples.use(express.static("public/js/test/examples"));
examples.listen(7999, "0.0.0.0", function(err, result) {
  console.log("View debugger examples at http://localhost:7999");
});
{'fraction_non_alphanumeric': 0.18500424808836025, 'fraction_numerical': 0.012744265080713678, 'mean_word_length': 4.456546929316338, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2940617', 'n_tokens_mistral': 1536, 'n_tokens_neox': 1489, 'n_words': 306}