<commit_before>var http = require("http");
var fs = require("fs");
var endsWith = require('ends-with');
var startsWith = require("starts-with")
var stringContains = require("string-contains");
var processJavaScriptCode = require("./src/compilation/processJavaScriptCode")
import { replaceJSScriptTags } from "./src/getJSScriptTags"
var _ = require("underscore")

http.createServer(handleRequest).listen(7500)
var isDev = process.argv[2] === "dev"

function handleRequest(request, response){
    var path = request.url.split("?")[0]
    var isInternalRequest = startsWith(path, "/fromjs-internals/")

    if (isInternalRequest){
        if (endsWith(path, "fromjs.css")) {
            path = require("path").normalize(__dirname + "/../fromjs.css")
        }

        path = path.replace("/fromjs-internals/", __dirname + "/../" + "dist/")


    } else {
        path = "." + path
    }

    console.log("Request for", request.url)
    if (endsWith(path, ".js.map") && !isInternalRequest){
        path = path.substr(0, path.length - ".map".length)
    }
    if (endsWith(path, ".dontprocess")){
        path = path.substr(0, path.length - ".dontprocess".length)
    }


    if (fs.existsSync(path)){
        if (fs.lstatSync(path).isDirectory()){
            var files = fs.readdirSync(path);
            response.write("<html>")
            files.forEach(function(file){
                response.write("<a href='" + encodeURI("/" + path + (endsWith(path, "/") ? "" : "/") + file)  +"'>" + _.escape(file) + "</a><br/>")
            })
            response.write("</html>")
        } else {
            var fileContents = fs.readFileSync(path).toString()

            if (endsWith(request.url, ".html")){
                var originalHtmlScriptTag = "<script id='fromjs-initial-html' html-filename='" + request.url + "' type='text/template'>" + encodeURIComponent(fileContents) + "</script>"
                var fromJSUrl = "/fromjs-internals/from.js"
                // not using webpack dev server for now to avoid having to start lots of webpack servers
                // if (isDev){
                //     fromJSUrl = "http://localhost:8080/dist/from.js"
                // }
                var scriptTagHtml = '<script src="' + fromJSUrl + '" charset="utf-8"></script>'
                var linkTagHtml = '<link rel="stylesheet" href="' + "/fromjs-internals/fromjs.css" + '"/>'
                var insertedHtml = originalHtmlScriptTag + scriptTagHtml + linkTagHtml

                var uid = 1;
                fileContents = replaceJSScriptTags(fileContents, function(content) {
                    var encodedScript = encodeURI(content)
                    return `
                        var script = originalCreateElement.call(document, "script")
                        script.text = decodeURI("${encodedScript}")
                        document.body.appendChild(script)
                    `
                })

                // we need a  proper solution for processing initial page html... but for now this has to do
                var script = "<script>makeSureInitialHTMLHasBeenProcessed()</script>"
                var hasBody = false;
                fileContents = fileContents.replace(/<body[\w\W]*?>[\w\W]*?<\/body>/, function(match){
                    var hasScriptTag = false;
                    hasBody = true;

                    match = match.replace(/<script[\w\W]*?>[\w\W]*?<\/script>/g, function(scriptMatch){
                        if (scriptMatch.split(">")[0].indexOf("template") !== -1) {
                            // skip script tag that look like templates
                            return scriptMatch
                        }
                        if (hasScriptTag) {return scriptMatch}
                        hasScriptTag = true;
                        return script + scriptMatch
                    })
                    if (!hasScriptTag) {
                        match = match.replace(/<\/body>$/, script + "</body>")
                    }
                    return match
                })
                if (!hasBody) {
                    fileContents += script
                }

                if (stringContains(fileContents, "<head>")){
                    fileContents = fileContents.replace("<head>", "<head>" + insertedHtml)
                } else {
                    fileContents = insertedHtml + fileContents
                }
            }

            if (endsWith(request.url, ".html.dontprocess")){
                // nothing i need to do actually
            }

            if ((endsWith(request.url, ".js") || endsWith(request.url, ".js.map")) &&
                !stringContains(request.url, "/dontprocess") &&
                !stringContains(request.url, "/vis") &&
                !stringContains(request.url, ".dontprocess") &&
                !isInternalRequest
            ) {
                var jsFileName = path.split("/")[path.split("/").length - 1]
                if (endsWith(request.url, ".js")) {
                    var res = processJavaScriptCode(fileContents)
                    fileContents = res.code
                    fileContents += "\n//# sourceMappingURL=" + jsFileName + ".map"

                }
                if (endsWith(request.url, ".js.map") ){
                    fileContents = JSON.stringify(processJavaScriptCode(fileContents, {filename: jsFileName}).map, null, 4)
                }
<commit_msg>Say what URL to go to after server.<commit_after>var http = require("http");
var fs = require("fs");
var endsWith = require('ends-with');
var startsWith = require("starts-with")
var stringContains = require("string-contains");
var processJavaScriptCode = require("./src/compilation/processJavaScriptCode")
import { replaceJSScriptTags } from "./src/getJSScriptTags"
var _ = require("underscore")

http.createServer(handleRequest).listen(7500)
console.log("Open localhost:7500 in your browser and open an HTML file from there")
var isDev = process.argv[2] === "dev"

function handleRequest(request, response){
    var path = request.url.split("?")[0]
    var isInternalRequest = startsWith(path, "/fromjs-internals/")

    if (isInternalRequest){
        if (endsWith(path, "fromjs.css")) {
            path = require("path").normalize(__dirname + "/../fromjs.css")
        }

        path = path.replace("/fromjs-internals/", __dirname + "/../" + "dist/")


    } else {
        path = "." + path
    }

    console.log("Request for", request.url)
    if (endsWith(path, ".js.map") && !isInternalRequest){
        path = path.substr(0, path.length - ".map".length)
    }
    if (endsWith(path, ".dontprocess")){
        path = path.substr(0, path.length - ".dontprocess".length)
    }


    if (fs.existsSync(path)){
        if (fs.lstatSync(path).isDirectory()){
            var files = fs.readdirSync(path);
            response.write("<html>")
            files.forEach(function(file){
                response.write("<a href='" + encodeURI("/" + path + (endsWith(path, "/") ? "" : "/") + file)  +"'>" + _.escape(file) + "</a><br/>")
            })
            response.write("</html>")
        } else {
            var fileContents = fs.readFileSync(path).toString()

            if (endsWith(request.url, ".html")){
                var originalHtmlScriptTag = "<script id='fromjs-initial-html' html-filename='" + request.url + "' type='text/template'>" + encodeURIComponent(fileContents) + "</script>"
                var fromJSUrl = "/fromjs-internals/from.js"
                // not using webpack dev server for now to avoid having to start lots of webpack servers
                // if (isDev){
                //     fromJSUrl = "http://localhost:8080/dist/from.js"
                // }
                var scriptTagHtml = '<script src="' + fromJSUrl + '" charset="utf-8"></script>'
                var linkTagHtml = '<link rel="stylesheet" href="' + "/fromjs-internals/fromjs.css" + '"/>'
                var insertedHtml = originalHtmlScriptTag + scriptTagHtml + linkTagHtml

                var uid = 1;
                fileContents = replaceJSScriptTags(fileContents, function(content) {
                    var encodedScript = encodeURI(content)
                    return `
                        var script = originalCreateElement.call(document, "script")
                        script.text = decodeURI("${encodedScript}")
                        document.body.appendChild(script)
                    `
                })

                // we need a  proper solution for processing initial page html... but for now this has to do
                var script = "<script>makeSureInitialHTMLHasBeenProcessed()</script>"
                var hasBody = false;
                fileContents = fileContents.replace(/<body[\w\W]*?>[\w\W]*?<\/body>/, function(match){
                    var hasScriptTag = false;
                    hasBody = true;

                    match = match.replace(/<script[\w\W]*?>[\w\W]*?<\/script>/g, function(scriptMatch){
                        if (scriptMatch.split(">")[0].indexOf("template") !== -1) {
                            // skip script tag that look like templates
                            return scriptMatch
                        }
                        if (hasScriptTag) {return scriptMatch}
                        hasScriptTag = true;
                        return script + scriptMatch
                    })
                    if (!hasScriptTag) {
                        match = match.replace(/<\/body>$/, script + "</body>")
                    }
                    return match
                })
                if (!hasBody) {
                    fileContents += script
                }

                if (stringContains(fileContents, "<head>")){
                    fileContents = fileContents.replace("<head>", "<head>" + insertedHtml)
                } else {
                    fileContents = insertedHtml + fileContents
                }
            }

            if (endsWith(request.url, ".html.dontprocess")){
                // nothing i need to do actually
            }

            if ((endsWith(request.url, ".js") || endsWith(request.url, ".js.map")) &&
                !stringContains(request.url, "/dontprocess") &&
                !stringContains(request.url, "/vis") &&
                !stringContains(request.url, ".dontprocess") &&
                !isInternalRequest
            ) {
                var jsFileName = path.split("/")[path.split("/").length - 1]
                if (endsWith(request.url, ".js")) {
                    var res = processJavaScriptCode(fileContents)
                    fileContents = res.code
                    fileContents += "\n//# sourceMappingURL=" + jsFileName + ".map"

                }
                if (endsWith(request.url, ".js.map") ){
                    fileContents = JSON.stringify(processJavaScriptCode(fileContents, {filename: jsFileName}).map, null, 4)
                }
{'fraction_non_alphanumeric': 0.1407150041144738, 'fraction_numerical': 0.003657310048459358, 'mean_word_length': 1.6881297616121897, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '439054', 'n_tokens_mistral': 2936, 'n_tokens_neox': 2757, 'n_words': 595}