<commit_before>#!/usr/bin/env coffee

# serv.ps
# http server for cobalt.

fs      = require 'fs'
child_process = require 'child_process'
os      = require 'os'

express = require 'express'
request = require 'request'
exec    = require('child_process').exec
mongoose = require 'mongoose'

User = require 'models/user'
Box = require 'models/box'
SSHKey = require 'models/ssh_key'

app = express()

app.use express.bodyParser()

app.configure 'staging', ->
  app.use express.logger()
  app.use express.errorHandler { dumpExceptions: true, showStack: true}

app.configure 'production', ->
  app.use express.logger()
  app.use express.errorHandler()

mongoose.connect process.env['COBALT_DB']

server_hostname = ''
root_url = ''
app.all "*", (req, res, next) ->
  server_hostname = req.header('host')
  root_url = "http://#{server_hostname}"
  root_url = root_url + ":#{process.env.COBALT_PORT}" unless process.env.COBALT_PORT == '8000'
  next()

# Templating language
app.set('view engine', 'ejs')

# TODO: should this be middleware?
# Check if scraperwiki currently recognises the api key
# If fails, check if that apikey has been valid in the past for box creation
check_api_key = (req, res, next) ->
  res.header('Content-Type', 'application/json')
  apikey = req.body.apikey or req.query.apikey
  if apikey?
    url = "https://scraperwiki.com/froth/check_key/#{apikey}"
    request.get url, (err, resp, body) ->
      body = JSON.parse body
      if resp.statusCode is 200 and req.params.org is body.org
        return next()
      else
        return res.send {error: "Unauthorised"}, 403
  else
    return res.send {error: "No API key supplied"}, 403


# GET REQUESTS
# These should all be idempotent, i.e. make no changes to the server.

app.get "/", (req, res) ->
  res.header('Content-Type', 'application/json')
  res.render('index', {rooturl: root_url})

# Documentation for SSHing to a box.
app.get "/:org/:project/?", (req, res) ->
  box_name = req.params.org + '/' + req.params.project
  res.header('Content-Type', 'application/json')
  Box.findOne {name: box_name}, (err, box) ->
    return res.send { error: "Box not found" }, 404 unless box?
    res.render 'box',
      box_name: box_name
      rooturl: root_url
      server_hostname: server_hostname

# Get scraperwiki.json settings file
app.get "/:org/:project/settings/?", check_api_key
app.get "/:org/:project/settings/?", (req, res) ->
  res.header('Content-Type', 'application/json')
  box_name = req.params.org + '/' + req.params.project
  fs.readFile "/home/#{box_name}/scraperwiki.json",
    'utf8',
    (err, data) ->
      if !err
        res.send data
      else
        res.send {error:"scraperwiki.json not found"}, 404

# Get file
app.get "/:org/:project/files/*", check_api_key
app.get "/:org/:project/files/*", (req, res) ->
  res.header('Content-Type', 'application/json')
  box_name = req.params.org + '/' + req.params.project
  path = req.path.replace "/#{box_name}/files", ''
  child_process.exec "su -c \"cat /home#{path}\" #{box_name}",
    'utf8',
    (err, stdout, stderr) ->
      if !err and !stderr
        res.send stdout
      else
        res.send {error:"#{path} is not a file"}, 400


# POST REQUESTS
# These should make changes somewhere, likely to the mongodb database

# Check API key for all org/project URLs
app.post "/:org*", check_api_key

# Create a box
app.post "/:org/:project/?", (req, res) ->
  res.header('Content-Type', 'application/json')
  box_name = req.params.org + '/' + req.params.project
  re = /^[a-zA-Z0-9_+-]+\/[a-zA-Z0-9_+-]+$/
  if not re.test box_name
    return res.send {error:
      "Box name should match the regular expression #{String(re)}"}, 404
  new User({apikey: req.body.apikey}).save()
  User.findOne {apikey: req.body.apikey}, (err, user) ->
    return res.send {error: "User not found" }, 404 unless user?
    Box.findOne {name: box_name}, (err, box) ->
      if box
        return res.send {error: "Box already exists"}
      else
        new Box({user: user._id, name: box_name}).save (err) ->
          if err
            console.log "Creating box: #{err} "
            return res.send {error: "Unknown error"}
          else
            exports.unix_user_add box_name, (err, stdout, stderr) ->
              any_stderr = stderr is not ''
              console.log "Error adding user: #{err} #{stderr}" if err? or any_stderr
              return res.send {error: "Unable to create box"} if err? or any_stderr
              return res.send {status: "ok"}

# Add an SSH key to a box
app.post "/:org/:project/sshkeys/?", (req, res) ->
  box_name = req.params.org + '/' + req.params.project

  res.header('Content-Type', 'application/json')
  unless req.body.sshkey? then return res.send { error: "SSH Key not specified" }, 400

  Box.findOne {name: box_name}, (err, box) ->
    return res.send { error: "Box not found" }, 404 unless box?
    User.findOne {apikey: req.body.apikey}, (err, user) ->
      return res.send { error: "Unauthorised" }, 403 unless user?
      return res.send { error: "Unauthorised" }, 403 unless user._id.toString() == box.user.toString()
      try
        name = SSHKey.extract_name req.body.sshkey
      catch TypeError
        return res.send { error: "SSH Key format not valid" }, 400
      unless name then return res.send { error: "SSH Key has no name" }, 400
      key = new SSHKey
        box: box._id
        name: name
        key: req.body.sshkey

      key.save ->
        SSHKey.find {box: box._id}, (err, sshkeys) ->
          keys_path = "/opt/cobalt/etc/sshkeys/#{box.name}/authorized_keys"

          keys = for key in sshkeys
            "#{key.key}"

          fs.writeFileSync keys_path, keys.join '\n', 'utf8'
          # Note: octal.  This is deliberate.
          fs.chmodSync keys_path, 0o600
          child_process.exec "chown #{box.name}: #{keys_path}" # insecure
          return res.send {"status": "ok"}

# Set scraperwiki.json settings file
app.post "/:org/:project/settings/?", (req, res) ->
  box_name = req.params.org + '/' + req.params.project

  json = null
  try
    json = JSON.parse req.body.data
  catch e
    return res.send { error: "Invalid JSON" }, 400
  fs.writeFile "/home/#{box_name}/scraperwiki.json",
    (JSON.stringify json, null, 2), 'utf8', (err) ->
      if !err
        return res.send { message: "ok" }, 200
      else
        return res.send { error: "Couldn't write scraperwiki.json" } , 400


app.listen process.env.COBALT_PORT

exports.unix_user_add = (box_name, callback) ->
  cmd = """
        cd /opt/cobalt &&
        . ./code/box_lib.sh &&
        create_user #{box_name} &&
        create_user_directories #{box_name} &&
        furnish_box #{box_name}
        """
  # insecure - sanitise box_name
  exec cmd, callback
<commit_msg>make the file read api a tad more secure<commit_after>#!/usr/bin/env coffee

# serv.ps
# http server for cobalt.

fs      = require 'fs'
child_process = require 'child_process'
os      = require 'os'

express = require 'express'
request = require 'request'
exec    = require('child_process').exec
mongoose = require 'mongoose'

User = require 'models/user'
Box = require 'models/box'
SSHKey = require 'models/ssh_key'

app = express()

app.use express.bodyParser()

app.configure 'staging', ->
  app.use express.logger()
  app.use express.errorHandler { dumpExceptions: true, showStack: true}

app.configure 'production', ->
  app.use express.logger()
  app.use express.errorHandler()

mongoose.connect process.env['COBALT_DB']

server_hostname = ''
root_url = ''
app.all "*", (req, res, next) ->
  server_hostname = req.header('host')
  root_url = "http://#{server_hostname}"
  root_url = root_url + ":#{process.env.COBALT_PORT}" unless process.env.COBALT_PORT == '8000'
  next()

# Templating language
app.set('view engine', 'ejs')

# TODO: should this be middleware?
# Check if scraperwiki currently recognises the api key
# If fails, check if that apikey has been valid in the past for box creation
check_api_key = (req, res, next) ->
  res.header('Content-Type', 'application/json')
  apikey = req.body.apikey or req.query.apikey
  if apikey?
    url = "https://scraperwiki.com/froth/check_key/#{apikey}"
    request.get url, (err, resp, body) ->
      body = JSON.parse body
      if resp.statusCode is 200 and req.params.org is body.org
        return next()
      else
        return res.send {error: "Unauthorised"}, 403
  else
    return res.send {error: "No API key supplied"}, 403


# GET REQUESTS
# These should all be idempotent, i.e. make no changes to the server.

app.get "/", (req, res) ->
  res.header('Content-Type', 'application/json')
  res.render('index', {rooturl: root_url})

# Documentation for SSHing to a box.
app.get "/:org/:project/?", (req, res) ->
  box_name = req.params.org + '/' + req.params.project
  res.header('Content-Type', 'application/json')
  Box.findOne {name: box_name}, (err, box) ->
    return res.send { error: "Box not found" }, 404 unless box?
    res.render 'box',
      box_name: box_name
      rooturl: root_url
      server_hostname: server_hostname

# Get scraperwiki.json settings file
app.get "/:org/:project/settings/?", check_api_key
app.get "/:org/:project/settings/?", (req, res) ->
  res.header('Content-Type', 'application/json')
  box_name = req.params.org + '/' + req.params.project
  path = path.replace /\'/g, '' # Sanitise path?
  box_name = box_name.replace /\'/g, '' # Sanitise box name?
  fs.readFile "/home/#{box_name}/scraperwiki.json",
    'utf8',
    (err, data) ->
      if !err
        res.send data
      else
        res.send {error:"scraperwiki.json not found"}, 404

# Get file
app.get "/:org/:project/files/*", check_api_key
app.get "/:org/:project/files/*", (req, res) ->
  res.header('Content-Type', 'application/json')
  box_name = req.params.org + '/' + req.params.project
  path = req.path.replace "/#{box_name}/files", ''
  child_process.exec "su -c \"cat /home#{path}\" #{box_name}",
    'utf8',
    (err, stdout, stderr) ->
      if !err and !stderr
        res.send stdout
      else
        res.send {error:"#{path} is not a file"}, 400


# POST REQUESTS
# These should make changes somewhere, likely to the mongodb database

# Check API key for all org/project URLs
app.post "/:org*", check_api_key

# Create a box
app.post "/:org/:project/?", (req, res) ->
  res.header('Content-Type', 'application/json')
  box_name = req.params.org + '/' + req.params.project
  re = /^[a-zA-Z0-9_+-]+\/[a-zA-Z0-9_+-]+$/
  if not re.test box_name
    return res.send {error:
      "Box name should match the regular expression #{String(re)}"}, 404
  new User({apikey: req.body.apikey}).save()
  User.findOne {apikey: req.body.apikey}, (err, user) ->
    return res.send {error: "User not found" }, 404 unless user?
    Box.findOne {name: box_name}, (err, box) ->
      if box
        return res.send {error: "Box already exists"}
      else
        new Box({user: user._id, name: box_name}).save (err) ->
          if err
            console.log "Creating box: #{err} "
            return res.send {error: "Unknown error"}
          else
            exports.unix_user_add box_name, (err, stdout, stderr) ->
              any_stderr = stderr is not ''
              console.log "Error adding user: #{err} #{stderr}" if err? or any_stderr
              return res.send {error: "Unable to create box"} if err? or any_stderr
              return res.send {status: "ok"}

# Add an SSH key to a box
app.post "/:org/:project/sshkeys/?", (req, res) ->
  box_name = req.params.org + '/' + req.params.project

  res.header('Content-Type', 'application/json')
  unless req.body.sshkey? then return res.send { error: "SSH Key not specified" }, 400

  Box.findOne {name: box_name}, (err, box) ->
    return res.send { error: "Box not found" }, 404 unless box?
    User.findOne {apikey: req.body.apikey}, (err, user) ->
      return res.send { error: "Unauthorised" }, 403 unless user?
      return res.send { error: "Unauthorised" }, 403 unless user._id.toString() == box.user.toString()
      try
        name = SSHKey.extract_name req.body.sshkey
      catch TypeError
        return res.send { error: "SSH Key format not valid" }, 400
      unless name then return res.send { error: "SSH Key has no name" }, 400
      key = new SSHKey
        box: box._id
        name: name
        key: req.body.sshkey

      key.save ->
        SSHKey.find {box: box._id}, (err, sshkeys) ->
          keys_path = "/opt/cobalt/etc/sshkeys/#{box.name}/authorized_keys"

          keys = for key in sshkeys
            "#{key.key}"

          fs.writeFileSync keys_path, keys.join '\n', 'utf8'
          # Note: octal.  This is deliberate.
          fs.chmodSync keys_path, 0o600
          child_process.exec "chown #{box.name}: #{keys_path}" # insecure
          return res.send {"status": "ok"}

# Set scraperwiki.json settings file
app.post "/:org/:project/settings/?", (req, res) ->
  box_name = req.params.org + '/' + req.params.project

  json = null
  try
    json = JSON.parse req.body.data
  catch e
    return res.send { error: "Invalid JSON" }, 400
  fs.writeFile "/home/#{box_name}/scraperwiki.json",
    (JSON.stringify json, null, 2), 'utf8', (err) ->
      if !err
        return res.send { message: "ok" }, 200
      else
        return res.send { error: "Couldn't write scraperwiki.json" } , 400


app.listen process.env.COBALT_PORT

exports.unix_user_add = (box_name, callback) ->
  cmd = """
        cd /opt/cobalt &&
        . ./code/box_lib.sh &&
        create_user #{box_name} &&
        create_user_directories #{box_name} &&
        furnish_box #{box_name}
        """
  # insecure - sanitise box_name
  exec cmd, callback

{'fraction_non_alphanumeric': 0.14933021008710928, 'fraction_numerical': 0.00995534733914062, 'mean_word_length': 2.837640449438202, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 53, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '299327', 'n_tokens_mistral': 4733, 'n_tokens_neox': 4557, 'n_words': 1265}