<commit_before>'use strict';

var express = require('express');
var gm = require('gm');
var formidable = require('formidable');
var fs = require('fs-extra');
var config = require('config');
var AWS = require('aws-sdk');
var sqlite3 = require('sqlite3').verbose();
var responseTime = require('response-time')
var uuid = require('node-uuid');
var bodyParser = require('body-parser');
var db;

AWS.config.update({accessKeyId: config.get('aws.access_key'), secretAccessKey: config.get('aws.secret_key'), region: config.get('aws.region')});

// The AWS config needs to be set before this objectis created
var S3 = new AWS.S3();

// Re-use exisiting prepared queries
var insertImage;
var selectImage;
var insertToken;
var consumeToken;
var deleteOldTokens;

function prepareDb(callback) {
  db.serialize(function () {
    console.log("Creating the db schema");
    try {
      db.run("CREATE TABLE images (id VARCHAR(255), x INT(6), y INT(6), fit VARCHAR(8), file_type VARCHAR(8), url VARCHAR(255))");
      db.run("CREATE UNIQUE INDEX unique_image ON images(id,x,y,fit,file_type)");

      db.run("CREATE TABLE tokens ( id VARCHAR(255), image_id VARCHAR(255), valid_until TEXT, used INT(1))");
      db.run("CREATE UNIQUE INDEX unique_token ON tokens(id)");
      db.run("CREATE UNIQUE INDEX unique_image_request ON tokens(image_id)");
      db.run("CREATE INDEX token_date ON tokens(id, valid_until, used)");
      console.log("Doing the callback from prepareDb");
      callback();
    } catch (e) {
      console.error(e);
    }
  });
}

// Central logging. console.log can be replaced by writing to a logfile for example
function log(level, message) {
  var obj = {
    datetime: Date.now(),
    severity: level,
    message: message
  };
  console.log(JSON.stringify(obj));
}

function logRequest(req, res, time) {
  var remoteIp = req.headers['x-forwarded-for'] || req.ip;
  var obj = {
    datetime: Date.now(),
    method: req.method,
    url: req.url,
    client: remoteIp,
    response_time: (time / 1e3),
    response_status: res.statusCode
  };
  if (isGetRequest(req, res)) {
    if (res.statusCode === 200) {
      obj.cache_hit = false;
    } else if (res.statusCode === 307) {
      obj.cache_hit = true
    }
    var imageParams = getImageParams(req);
    for (var param in imageParams) {
      obj[param] = imageParams[param];
    }
  }
  console.log(JSON.stringify(obj));
}

function isGetRequest(req, res) {
  return req.url !== '/healthcheck' && req.method === 'GET';
}

function supportedFileType(fileType) {
  switch (fileType) {
    case 'jpg':
    case 'jpeg':
    case 'jfif':
    case 'jpe':
      return 'image/jpeg';
    case 'png':
      return 'image/png';
    default:
      return null;
  }
}

function isValidRequest(url) {
  return splitUrl(url) !== null;
}

function splitUrl(url) {
  return url.match(/^\/(.*)_(\d+)_(\d+)(_(\d+)x)?\.(.*)/);
}

function getImageParams(req) {
  var url = req.url;
  var queryParams = null;
  var split = url.indexOf('?');
  if (split > -1) {
    queryParams = url.substring(split + 1, url.length);
    url = url.substring(0, split);
  }

  var matches = splitUrl(url);
  var res;
  if (matches !== null) {
    res = {
      fileName: matches[1],
      resolutionX: parseInt(matches[2], 10),
      resolutionY: parseInt(matches[3], 10),
      fileType: matches[6].toLowerCase(),
      fit: 'clip'
    };

    if (matches[5] !== undefined) {
      res.resolutionX *= parseInt(matches[5], 10);
      res.resolutionY *= parseInt(matches[5], 10);
    }
  } else {
    matches = url.match(/^\/(.*)\.([^.]+)$/);
    res = {
      fileName: matches[1],
      fileType: matches[2].toLowerCase()
    };
  }

  if (queryParams === 'fit=crop') {
    res.fit = 'crop';
  }

  return res;
}

var image = {};
image.get = function (req, res) {
  if (!isValidRequest(req.url)) {
    // Invalid URL
    log('error', '404 Error for ' + res.url);
    res.writeHead(404, 'File not found');
    res.end();
    return;
  }

  var params = getImageParams(req);

  if (supportedFileType(params.fileType) === null) {
    log('error', 'Filetype ' + params.fileType + ' is not supported');
    res.writeHead(415, 'Unsupported media type');
    res.end();
    return;
  }

  if (params.resolutionX < 0 || params.resolutionY < 0 || params.resolutionX > config.get('constraints.max_width') || params.resolutionY > config.get('constraints.max_height')) {
    res.writeHead(400, 'Bad request');
    res.end('The requested image size falls outside of the allowed boundaries of this service.');
    return;
  }

  log('info', 'Requesting file ' + params.fileName + ' in ' + params.fileType + ' format in a ' + params.resolutionX + 'x' + params.resolutionY + 'px resolution');

  image.checkCacheOrCreate(params.fileName, params.fileType, params.resolutionX, params.resolutionY, params.fit, res);
};
image.checkCacheOrCreate = function (fileName, fileType, resolutionX, resolutionY, fit, res) {
  // Check if it exists in the cache
  selectImage.get([fileName, resolutionX, resolutionY, fit, supportedFileType(fileType)], function (err, data) {
    if (!err && data) {
      // It is in the cache, so redirect to there
      log('info', 'cache hit for ' + fileName + '.' + fileType + '(' + resolutionX + 'x' + resolutionY + 'px, fit: ' + fit + ')');
      res.writeHead(307, {'Location': data.url, 'Cache-Control': 'public'});
      res.end();
      return;
    }s

    // It does not exist in the cache, so generate and upload
    image.encodeAndUpload(fileName, fileType, resolutionX, resolutionY, fit, res);
  });
};
image.encodeAndUpload = function (fileName, fileType, resolutionX, resolutionY, fit, res) {
  var file = config.get('originals_dir') + '/' + fileName;
  fs.exists(file, function (exists) {
    if (!exists) {
      res.writeHead('404', 'File not found');<commit_msg>Fixed typo.<commit_after>'use strict';

var express = require('express');
var gm = require('gm');
var formidable = require('formidable');
var fs = require('fs-extra');
var config = require('config');
var AWS = require('aws-sdk');
var sqlite3 = require('sqlite3').verbose();
var responseTime = require('response-time')
var uuid = require('node-uuid');
var bodyParser = require('body-parser');
var db;

AWS.config.update({accessKeyId: config.get('aws.access_key'), secretAccessKey: config.get('aws.secret_key'), region: config.get('aws.region')});

// The AWS config needs to be set before this objectis created
var S3 = new AWS.S3();

// Re-use exisiting prepared queries
var insertImage;
var selectImage;
var insertToken;
var consumeToken;
var deleteOldTokens;

function prepareDb(callback) {
  db.serialize(function () {
    console.log("Creating the db schema");
    try {
      db.run("CREATE TABLE images (id VARCHAR(255), x INT(6), y INT(6), fit VARCHAR(8), file_type VARCHAR(8), url VARCHAR(255))");
      db.run("CREATE UNIQUE INDEX unique_image ON images(id,x,y,fit,file_type)");

      db.run("CREATE TABLE tokens ( id VARCHAR(255), image_id VARCHAR(255), valid_until TEXT, used INT(1))");
      db.run("CREATE UNIQUE INDEX unique_token ON tokens(id)");
      db.run("CREATE UNIQUE INDEX unique_image_request ON tokens(image_id)");
      db.run("CREATE INDEX token_date ON tokens(id, valid_until, used)");
      console.log("Doing the callback from prepareDb");
      callback();
    } catch (e) {
      console.error(e);
    }
  });
}

// Central logging. console.log can be replaced by writing to a logfile for example
function log(level, message) {
  var obj = {
    datetime: Date.now(),
    severity: level,
    message: message
  };
  console.log(JSON.stringify(obj));
}

function logRequest(req, res, time) {
  var remoteIp = req.headers['x-forwarded-for'] || req.ip;
  var obj = {
    datetime: Date.now(),
    method: req.method,
    url: req.url,
    client: remoteIp,
    response_time: (time / 1e3),
    response_status: res.statusCode
  };
  if (isGetRequest(req, res)) {
    if (res.statusCode === 200) {
      obj.cache_hit = false;
    } else if (res.statusCode === 307) {
      obj.cache_hit = true
    }
    var imageParams = getImageParams(req);
    for (var param in imageParams) {
      obj[param] = imageParams[param];
    }
  }
  console.log(JSON.stringify(obj));
}

function isGetRequest(req, res) {
  return req.url !== '/healthcheck' && req.method === 'GET';
}

function supportedFileType(fileType) {
  switch (fileType) {
    case 'jpg':
    case 'jpeg':
    case 'jfif':
    case 'jpe':
      return 'image/jpeg';
    case 'png':
      return 'image/png';
    default:
      return null;
  }
}

function isValidRequest(url) {
  return splitUrl(url) !== null;
}

function splitUrl(url) {
  return url.match(/^\/(.*)_(\d+)_(\d+)(_(\d+)x)?\.(.*)/);
}

function getImageParams(req) {
  var url = req.url;
  var queryParams = null;
  var split = url.indexOf('?');
  if (split > -1) {
    queryParams = url.substring(split + 1, url.length);
    url = url.substring(0, split);
  }

  var matches = splitUrl(url);
  var res;
  if (matches !== null) {
    res = {
      fileName: matches[1],
      resolutionX: parseInt(matches[2], 10),
      resolutionY: parseInt(matches[3], 10),
      fileType: matches[6].toLowerCase(),
      fit: 'clip'
    };

    if (matches[5] !== undefined) {
      res.resolutionX *= parseInt(matches[5], 10);
      res.resolutionY *= parseInt(matches[5], 10);
    }
  } else {
    matches = url.match(/^\/(.*)\.([^.]+)$/);
    res = {
      fileName: matches[1],
      fileType: matches[2].toLowerCase()
    };
  }

  if (queryParams === 'fit=crop') {
    res.fit = 'crop';
  }

  return res;
}

var image = {};
image.get = function (req, res) {
  if (!isValidRequest(req.url)) {
    // Invalid URL
    log('error', '404 Error for ' + res.url);
    res.writeHead(404, 'File not found');
    res.end();
    return;
  }

  var params = getImageParams(req);

  if (supportedFileType(params.fileType) === null) {
    log('error', 'Filetype ' + params.fileType + ' is not supported');
    res.writeHead(415, 'Unsupported media type');
    res.end();
    return;
  }

  if (params.resolutionX < 0 || params.resolutionY < 0 || params.resolutionX > config.get('constraints.max_width') || params.resolutionY > config.get('constraints.max_height')) {
    res.writeHead(400, 'Bad request');
    res.end('The requested image size falls outside of the allowed boundaries of this service.');
    return;
  }

  log('info', 'Requesting file ' + params.fileName + ' in ' + params.fileType + ' format in a ' + params.resolutionX + 'x' + params.resolutionY + 'px resolution');

  image.checkCacheOrCreate(params.fileName, params.fileType, params.resolutionX, params.resolutionY, params.fit, res);
};
image.checkCacheOrCreate = function (fileName, fileType, resolutionX, resolutionY, fit, res) {
  // Check if it exists in the cache
  selectImage.get([fileName, resolutionX, resolutionY, fit, supportedFileType(fileType)], function (err, data) {
    if (!err && data) {
      // It is in the cache, so redirect to there
      log('info', 'cache hit for ' + fileName + '.' + fileType + '(' + resolutionX + 'x' + resolutionY + 'px, fit: ' + fit + ')');
      res.writeHead(307, {'Location': data.url, 'Cache-Control': 'public'});
      res.end();
      return;
    }

    // It does not exist in the cache, so generate and upload
    image.encodeAndUpload(fileName, fileType, resolutionX, resolutionY, fit, res);
  });
};
image.encodeAndUpload = function (fileName, fileType, resolutionX, resolutionY, fit, res) {
  var file = config.get('originals_dir') + '/' + fileName;
  fs.exists(file, function (exists) {
    if (!exists) {
      res.writeHead('404', 'File not found');
{'fraction_non_alphanumeric': 0.16217843930138517, 'fraction_numerical': 0.011873010410393186, 'mean_word_length': 3.4656165962351135, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7071897', 'n_tokens_mistral': 3882, 'n_tokens_neox': 3775, 'n_words': 983}