<commit_before>// this is the request handler for the site.

module.exports = site

var router = require("./router.js")
, decorate = require('./decorate.js')
, config = require("./config.js")
, url = require("url")
, path = require("path")

, Keygrip = require("keygrip")

, StringDecoder = require('string_decoder').StringDecoder
, qs = require("querystring")


config.keys = new Keygrip(config.keys)

function site (req, res) {
  // only allow access via the canonical hostname
  if (config.canon(req, res)) return

  decorate(req, res, config)

  // kludge!  remove this once we figure out wtf is going on.
  var timeout = setTimeout(function () {
    res.log.error('TIMEOUT! %s', req.url, res._header)
    var er = new Error('Request timed out')
    res.error(500, er)
    setTimeout(function () {
      res.socket.destroy()
      res.emit('error', er)
    }, 5000)
  }, 30*1000) // nothing should take 30s ever.

  // maybe the issue is that it's sometimes not flushing?
  // flush after 2s, that might make it go away?
  var flusher = setTimeout(function () {
    res._flush()
  })

  res.on('finish', function () {
    clearTimeout(timeout)
    clearTimeout(flusher)
  })

  var parsed = url.parse(req.url)
  var pathname = parsed.pathname
  var normalPathname = path.normalize(pathname).replace(/\\/g, '/');

  // multiple //// chars in the path are stupid and should not be,<commit_msg>More flush/close/finish timeout error handling<commit_after>// this is the request handler for the site.

module.exports = site

var router = require("./router.js")
, decorate = require('./decorate.js')
, config = require("./config.js")
, url = require("url")
, path = require("path")

, Keygrip = require("keygrip")

, StringDecoder = require('string_decoder').StringDecoder
, qs = require("querystring")


config.keys = new Keygrip(config.keys)

function site (req, res) {
  // only allow access via the canonical hostname
  if (config.canon(req, res)) return

  decorate(req, res, config)

  // kludge!  remove this once we figure out wtf is going on.
  var timeout = setTimeout(function () {
    res.log.error('TIMEOUT! %s', req.url, res._header)
    var er = new Error('Request timed out')
    res.error(500, er)
    setTimeout(function () {
      res.socket.destroy()
      throw er
    }, 5000)
  }, 30*1000) // nothing should take 30s ever.

  // maybe the issue is that it's sometimes not flushing?
  // flush after 2s, that might make it go away?
  var flusher = setTimeout(function () {
    console.error('flusher!', req.url, res.statusCode, res.headers)
    res._flush()
  }, 2000)

  res.on('finish', timeoutOk)
  res.on('close', timeoutOk)
  function timeoutOk () {
    clearTimeout(timeout)
    clearTimeout(flusher)
  }

  var parsed = url.parse(req.url)
  var pathname = parsed.pathname
  var normalPathname = path.normalize(pathname).replace(/\\/g, '/');

  // multiple //// chars in the path are stupid and should not be,
{'fraction_non_alphanumeric': 0.14275979557069846, 'fraction_numerical': 0.012265758091993186, 'mean_word_length': 3.395209580838323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7257105', 'n_tokens_mistral': 968, 'n_tokens_neox': 924, 'n_words': 276}