<commit_before>if (process.env.NODE_ENV === "production") {
    require("@google/cloud-debug");
    require("@google/cloud-trace").start();
}

const url = require("url");
const path = require("path");
const moment = require("moment");
const express = require("express");
const exphbs = require("express-handlebars");
const emojiFlags = require("emoji-flags");
const taskQueue = require("./taskqueue");
const store = require("./store");

function fullUrl(req, path) {
    let baseUrl = process.env.BASE_URL;
    return url.resolve(baseUrl, url.resolve(req.baseUrl, path));
}

function mergeAndClean(...objs) {
    const result = {};
    objs.forEach(obj => {
        Object.keys(obj).forEach(key => {
            if (obj[key] !== undefined) {
                result[key] = obj[key];
            } else {
                delete result[key];
            }
        });
    });
    return result;
}

function extractInfo(req) {
    return mergeAndClean({
        ip: req.get("x-appengine-user-ip") || req.ip,
        country: req.get("x-appengine-country"),
        userAgent: req.get("user-agent")
    });
}

function formatTimestamp(ts) {
    return moment(ts).format();
}

const app = express();

// By default the express-handlebars package searches ./views for view templates
// and ./views/layouts for layouts.
app.engine("handlebars", exphbs({defaultLayout: "main"}));
app.set("view engine", "handlebars");

app.use("/assets", express.static(path.join(__dirname, "build")));

app.get("/", (req, res) => {
    res.render("index", {
        styles: ["/assets/common.css"],
        scripts: ["/assets/common.js"]
    });
});

app.get("/new", (req, res) => {
    store.create()
        .then(view => {
            res.redirect("/" + view);
        })
        .catch(err => {
            console.error(err);
            res.sendStatus(500);
        });
});

app.get("/:id", (req, res) => {<commit_msg>Collect referers<commit_after>if (process.env.NODE_ENV === "production") {
    require("@google/cloud-debug");
    require("@google/cloud-trace").start();
}

const url = require("url");
const path = require("path");
const moment = require("moment");
const express = require("express");
const exphbs = require("express-handlebars");
const emojiFlags = require("emoji-flags");
const taskQueue = require("./taskqueue");
const store = require("./store");

function fullUrl(req, path) {
    let baseUrl = process.env.BASE_URL;
    return url.resolve(baseUrl, url.resolve(req.baseUrl, path));
}

function mergeAndClean(...objs) {
    const result = {};
    objs.forEach(obj => {
        Object.keys(obj).forEach(key => {
            if (obj[key] !== undefined) {
                result[key] = obj[key];
            } else {
                delete result[key];
            }
        });
    });
    return result;
}

function extractInfo(req) {
    // Google App Engine passes in an approximate geolocation in header
    // "x-appengine-country" and uses code "ZZ" for unknown locations.
    let country = req.get("x-appengine-country");
    if (!country || country === "ZZ") {
        country = undefined;
    }

    return mergeAndClean({
        ip: req.get("x-appengine-user-ip") || req.ip,
        referrer: req.get("referrer"), // Express considers "referrer" and "referer" interchangeable
        country: country,
        userAgent: req.get("user-agent")
    });
}

function formatTimestamp(ts) {
    return moment(ts).format();
}

const app = express();

app.set("json spaces", 2);

// By default the express-handlebars package searches ./views for view templates
// and ./views/layouts for layouts.
app.engine("handlebars", exphbs({defaultLayout: "main"}));
app.set("view engine", "handlebars");

app.use("/assets", express.static(path.join(__dirname, "build")));

app.get("/", (req, res) => {
    res.render("index", {
        styles: ["/assets/common.css"],
        scripts: ["/assets/common.js"]
    });
});

app.get("/new", (req, res) => {
    store.create()
        .then(view => {
            res.redirect("/" + view);
        })
        .catch(err => {
            console.error(err);
            res.sendStatus(500);
        });
});

app.get("/:id.json", (req, res) => {
    const id = req.params.id;

    store.get(id)
        .then(item => {
            if (!item || !item.isView) {
                return res.sendStatus(404);
            }

            return store.list(item.other).then(entities => {
                const visits = entities.map(entity => {
                    return mergeAndClean(entity.info, {
                        timestamp: formatTimestamp(entity.timestamp)
                    });
                });

                res.json({
                    trapUrl: fullUrl(req, item.other),
                    visits: visits
                });
            });
        })
        .catch(err => {
            console.error(err);
            res.sendStatus(500);
        });
});

app.get("/:id", (req, res) => {
{'fraction_non_alphanumeric': 0.17769682250556568, 'fraction_numerical': 0.0026310463468933415, 'mean_word_length': 2.3437077131258457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '383648', 'n_tokens_mistral': 1498, 'n_tokens_neox': 1476, 'n_words': 304}