<commit_before>var http = require('http')
  , fs = require('fs')
  , url = require('url')
  , d3 = require('d3')
  , tree = require('./tree')
  , request = require('request')
  , spawn = require('child_process').spawn

http.createServer(function (req, res) {
  var _url = url.parse(req.url, true)
    , path = _url.pathname

  if (path == '/index.html' || path == '/') {
    return fs.createReadStream(__dirname + '/index.html').pipe(res)
  } else if (path == '/tree.css') {
    res.writeHead(200, { 'Content-Type': 'text/css' })
    var less = spawn('./node_modules/less/bin/lessc', ['-x', 'tree.less'])
    less.stdout.pipe(res)
    less.stderr.pipe(process.stderr)
  } else if (path === '/documents.json') {
    return fs.createReadStream(__dirname + '/documents.json').pipe(res)
  } else if (path === '/tree-patch.json') {
    return fs.createReadStream(__dirname + '/patch.json').pipe(res)
  } else if (path === '/matt-tree.json') {
    request('https://gist.githubusercontent.com/nathanbowser/71f670f77defb15887da/raw/279ef40200750dbd4eac953814c7dabe5e6e1b8b/gistfile1.json').pipe(res)
  } else if (path == '/bundle.js') {
    res.writeHead(200, {'Content-Type': 'application/javascript'})
    return fs.createReadStream(__dirname + '/bundle.js').pipe(res)
  } else if (path == '/sms-tree.json') {
    return fs.createReadStream(__dirname + '/tree.json').pipe(res)
  } else if (path == '/tree.json') {
    res.end(JSON.stringify(d3.layout.tree().nodes(tree(_url.query.depth || 5)).map(function (n) {
      n.parentId = n.parent && n.parent.id
      delete n.children
      delete n.parent
      return n
    })))
  } else {
    res.writeHead(404, {'Content-Type': 'text/plain'})
    res.write('Not found')
    res.end()
  }


}).listen(3000)
<commit_msg>adds a lot of documents endpoint<commit_after>var http = require('http')
  , fs = require('fs')
  , url = require('url')
  , d3 = require('d3')
  , tree = require('./tree')
  , request = require('request')
  , spawn = require('child_process').spawn

http.createServer(function (req, res) {
  var _url = url.parse(req.url, true)
    , path = _url.pathname

  if (path == '/index.html' || path == '/') {
    return fs.createReadStream(__dirname + '/index.html').pipe(res)
  } else if (path == '/tree.css') {
    res.writeHead(200, { 'Content-Type': 'text/css' })
    var less = spawn('./node_modules/less/bin/lessc', ['-x', 'tree.less'])
    less.stdout.pipe(res)
    less.stderr.pipe(process.stderr)
  } else if (path === '/lots-o-docs.json') {
    return request('https://gist.githubusercontent.com/nathanbowser/7eda0518120d7bac6847/raw/c7de51421ef571232f2a2acb690edc2ba8261fac/gistfile1.json').pipe(res)
  } else if (path === '/documents.json') {
    return fs.createReadStream(__dirname + '/documents.json').pipe(res)
  } else if (path === '/tree-patch.json') {
    return fs.createReadStream(__dirname + '/patch.json').pipe(res)
  } else if (path === '/matt-tree.json') {
    request('https://gist.githubusercontent.com/nathanbowser/71f670f77defb15887da/raw/279ef40200750dbd4eac953814c7dabe5e6e1b8b/gistfile1.json').pipe(res)
  } else if (path == '/bundle.js') {
    res.writeHead(200, {'Content-Type': 'application/javascript'})
    return fs.createReadStream(__dirname + '/bundle.js').pipe(res)
  } else if (path == '/sms-tree.json') {
    return fs.createReadStream(__dirname + '/tree.json').pipe(res)
  } else if (path == '/tree.json') {
    res.end(JSON.stringify(d3.layout.tree().nodes(tree(_url.query.depth || 5)).map(function (n) {
      n.parentId = n.parent && n.parent.id
      delete n.children
      delete n.parent
      return n
    })))
  } else {
    res.writeHead(404, {'Content-Type': 'text/plain'})
    res.write('Not found')
    res.end()
  }


}).listen(3000)

{'fraction_non_alphanumeric': 0.1930574098798398, 'fraction_numerical': 0.03791722296395193, 'mean_word_length': 3.590686274509804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1409226', 'n_tokens_mistral': 1416, 'n_tokens_neox': 1337, 'n_words': 221}