<commit_before>var fs = require('fs');
var exec = require('child_process').exec;

exec('mkdir -p data');

// 250 mb
var SIZE_LIMIT = 250 * 1024 * 1024 * 1024;

var LOG_FILENAME = 'logfile.txt'

// returns array of log entries {filename: "", index: ""}
function getLogEntries() {
  // If the file doesn't exist, create a blank one!
  if (!fs.existsSync(LOG_FILENAME)) {
    var fd = fs.openSync(LOG_FILENAME, "a");
    fs.closeSync(fd);
  }
  var logFile = fs.readFileSync(LOG_FILENAME, {encoding: "utf-8"});
  var entries = logFile.trim().split("\n");
  var log = [];
  for (var i in entries) {
    var entry = entries[i].split(" -%%- ");
    var logEntry = {filename: entry[0], index: i};
    log.push(logEntry);
  }
  return log;
}

// returns index
function appendToLogFile(filename, filesize) {
  fs.appendFileSync(LOG_FILENAME, filename + " -%%- " + filesize + "\n");
  return getLogEntries().length - 1;
}

// synchronous
exports.uploadFile = function(filedata) {
  if (!filedata) {
    return "Error!";
  }
  var filename = filedata.originalFilename;
  var path = filedata.path;
  var numBytes = filedata.size;

  if (!filename || !path) {
    return "Error!";
  }

  if (numBytes > SIZE_LIMIT) {
    return "Error!";
  }

  var entryIndex = appendToLogFile(filename, numBytes);
  exec('cp ' + path + ' data/' + entryIndex);
  console.log('uploaded: ' + filename + ' index: ' + entryIndex);
  return "index: " + entryIndex + ", url: store.haus/" + entryIndex;
}

function serveFile(id, filename, res) {
  res.setHeader('Content-disposition', 'attachment; filename=' + filename);
  var fileStream = fs.createReadStream('data/' + id);
  fileStream.pipe(res);
}

exports.getFile = function(req, res) {
  var id = req.params.id;
  if (!id) {
    res.status(404).send('File not found!');
    return;
  }

  var log = getLogEntries();
  if (id == 'head' | id == 'current') {
    if (log.length > 0) {
      id = log.length - 1;
    } else {
      id = -1;
    }
  }

  if (id >= 0 && id < log.length) {<commit_msg>update to host instead of store.haus by default<commit_after>var fs = require('fs');
var exec = require('child_process').exec;

exec('mkdir -p data');

// 250 mb
var SIZE_LIMIT = 250 * 1024 * 1024 * 1024;

var LOG_FILENAME = 'logfile.txt'

// returns array of log entries {filename: "", index: ""}
function getLogEntries() {
  // If the file doesn't exist, create a blank one!
  if (!fs.existsSync(LOG_FILENAME)) {
    var fd = fs.openSync(LOG_FILENAME, "a");
    fs.closeSync(fd);
  }
  var logFile = fs.readFileSync(LOG_FILENAME, {encoding: "utf-8"});
  var entries = logFile.trim().split("\n");
  var log = [];
  for (var i in entries) {
    var entry = entries[i].split(" -%%- ");
    var logEntry = {filename: entry[0], index: i};
    log.push(logEntry);
  }
  return log;
}

// returns index
function appendToLogFile(filename, filesize) {
  fs.appendFileSync(LOG_FILENAME, filename + " -%%- " + filesize + "\n");
  return getLogEntries().length - 1;
}

// synchronous
exports.uploadFile = function(filedata) {
  if (!filedata) {
    return "Error!";
  }
  var filename = filedata.originalFilename;
  var path = filedata.path;
  var numBytes = filedata.size;

  if (!filename || !path) {
    return "Error!";
  }

  if (numBytes > SIZE_LIMIT) {
    return "Error!";
  }

  var entryIndex = appendToLogFile(filename, numBytes);
  exec('cp ' + path + ' data/' + entryIndex);
  console.log('uploaded: ' + filename + ' index: ' + entryIndex);
  return "index: " + entryIndex + ", url: {host}/" + entryIndex;
}

function serveFile(id, filename, res) {
  res.setHeader('Content-disposition', 'attachment; filename=' + filename);
  var fileStream = fs.createReadStream('data/' + id);
  fileStream.pipe(res);
}

exports.getFile = function(req, res) {
  var id = req.params.id;
  if (!id) {
    res.status(404).send('File not found!');
    return;
  }

  var log = getLogEntries();
  if (id == 'head' | id == 'current') {
    if (log.length > 0) {
      id = log.length - 1;
    } else {
      id = -1;
    }
  }

  if (id >= 0 && id < log.length) {
{'fraction_non_alphanumeric': 0.17192118226600986, 'fraction_numerical': 0.013793103448275862, 'mean_word_length': 3.2837552742616034, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3251460', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1403, 'n_words': 357}