<commit_before>const config = require("./config").root();
const path = require("path");
const fs = require("fs");
const child_process = require("child_process");
const rimraf = require("rimraf");
const ua = require("universal-analytics");

// Convert to milliseconds:
let keepTime = config.exportKeepTime * 60 * 1000;

function exportPath(deviceId) {
  if (deviceId.search(/^[a-z0-9\-]+$/i) == -1) {
    reject(new Error("Bad deviceId"));
  }
  return path.join(config.exportBase, "export---" + deviceId);
}

function removeExportPath(dir) {
  return new Promise((resolve, reject) => {
    rimraf(dir, (error) => {
      if (error && error.code != "ENOENT") {
        reject(error);
      } else {
        resolve();
      }
    });
  });
}

function launchWget(deviceId) {
  return new Promise((resolve, reject) => {
    let dir = exportPath(deviceId);
    let script = path.join(__dirname, "export-shots.sh");
    let start = Date.now();
    let subprocess = child_process.spawn(
      script,
      {
        env: {
          BASE_DIR: dir,
          AUTH: require("./dbschema").getTextKeys()[0],
          DEVICE_ID: deviceId,
          PORT: config.port,
          CONTENT_PORT: config.contentPort,
        }
      }
    );
    subprocess.on("close", function (code) {
      console.info("Exported to", dir, "in", (Date.now() - start) + "ms", "exit code:", code);
    });
    subprocess.on("error", function (e) {
      console.warn("Error launching wget:", e);
    });
    resolve();
    // FIXME: I don't think an exception here will reject
    // Also maybe this is all synchronous
  });
}

function getWgetStatus(deviceId) {
  return new Promise((resolve, reject) => {
    let dir = exportPath(deviceId);
    let result = {errors: []};
    let steps = 3;
    function resolveIfDone() {
      if (steps <= 0) {
        resolve(result);
      }
    }
    fs.readFile(path.join(dir, "wget-process.pid"), {encoding: "UTF-8"}, (error, data) => {
      if (error && error.code == "ENOENT") {
        // File doesn't exist
        result.running = false;
      } else if (error) {
        result.errors.push(["Error loading process PID:", error]);
      } else {
        result.running = true;
      }
      steps--;
      resolveIfDone();
    });
    fs.readFile(path.join(dir, "script-exit-code.txt"), {encoding: "UTF-8"}, (error, data) => {
      if (error && error.code == "ENOENT") {
        // Do nothing
      } else if (error) {
        result.errors.push(["Error loading exit code:", error]);
      } else {
        let code = parseInt(data, 10);
        if (code) {
          result.success = false;
        } else {
          result.success = true;
        }
      }
      steps--;
      resolveIfDone();
    });
    fs.stat(path.join(dir, "pageshot-export.zip"), (error, stat) => {
      if (error && error.code == "ENOENT") {
        result.zip = false;
      } else if (error) {
        result.errors.push(["Error checking zip file:", error]);
      } else {
        result.zip = {
          size: stat.size,
          mtime: stat.mtime.getTime(),
          keepTime: keepTime
        };
      }
      steps--;
      resolveIfDone();
    });
  });
}

exports.cleanExports = function () {
  let start = Date.now();
  fs.readdir(config.exportBase, (error, files) => {
    if (error && error.code == "ENOENT") {
      console.info("EXPORT: Could not clean exports, directory", config.exportBase, "does not exist");
      return;
    }<commit_msg>Fixes #804 Only warn once per process that we couldn't find the exports directory to clean.<commit_after>const config = require("./config").root();
const path = require("path");
const fs = require("fs");
const child_process = require("child_process");
const rimraf = require("rimraf");
const ua = require("universal-analytics");

// Convert to milliseconds:
let keepTime = config.exportKeepTime * 60 * 1000;

function exportPath(deviceId) {
  if (deviceId.search(/^[a-z0-9\-]+$/i) == -1) {
    reject(new Error("Bad deviceId"));
  }
  return path.join(config.exportBase, "export---" + deviceId);
}

function removeExportPath(dir) {
  return new Promise((resolve, reject) => {
    rimraf(dir, (error) => {
      if (error && error.code != "ENOENT") {
        reject(error);
      } else {
        resolve();
      }
    });
  });
}

function launchWget(deviceId) {
  return new Promise((resolve, reject) => {
    let dir = exportPath(deviceId);
    let script = path.join(__dirname, "export-shots.sh");
    let start = Date.now();
    let subprocess = child_process.spawn(
      script,
      {
        env: {
          BASE_DIR: dir,
          AUTH: require("./dbschema").getTextKeys()[0],
          DEVICE_ID: deviceId,
          PORT: config.port,
          CONTENT_PORT: config.contentPort,
        }
      }
    );
    subprocess.on("close", function (code) {
      console.info("Exported to", dir, "in", (Date.now() - start) + "ms", "exit code:", code);
    });
    subprocess.on("error", function (e) {
      console.warn("Error launching wget:", e);
    });
    resolve();
    // FIXME: I don't think an exception here will reject
    // Also maybe this is all synchronous
  });
}

function getWgetStatus(deviceId) {
  return new Promise((resolve, reject) => {
    let dir = exportPath(deviceId);
    let result = {errors: []};
    let steps = 3;
    function resolveIfDone() {
      if (steps <= 0) {
        resolve(result);
      }
    }
    fs.readFile(path.join(dir, "wget-process.pid"), {encoding: "UTF-8"}, (error, data) => {
      if (error && error.code == "ENOENT") {
        // File doesn't exist
        result.running = false;
      } else if (error) {
        result.errors.push(["Error loading process PID:", error]);
      } else {
        result.running = true;
      }
      steps--;
      resolveIfDone();
    });
    fs.readFile(path.join(dir, "script-exit-code.txt"), {encoding: "UTF-8"}, (error, data) => {
      if (error && error.code == "ENOENT") {
        // Do nothing
      } else if (error) {
        result.errors.push(["Error loading exit code:", error]);
      } else {
        let code = parseInt(data, 10);
        if (code) {
          result.success = false;
        } else {
          result.success = true;
        }
      }
      steps--;
      resolveIfDone();
    });
    fs.stat(path.join(dir, "pageshot-export.zip"), (error, stat) => {
      if (error && error.code == "ENOENT") {
        result.zip = false;
      } else if (error) {
        result.errors.push(["Error checking zip file:", error]);
      } else {
        result.zip = {
          size: stat.size,
          mtime: stat.mtime.getTime(),
          keepTime: keepTime
        };
      }
      steps--;
      resolveIfDone();
    });
  });
}

let warnedAboutExportsDirectory = false;

exports.cleanExports = function () {
  let start = Date.now();
  fs.readdir(config.exportBase, (error, files) => {
    if (error && error.code == "ENOENT") {
      if (!warnedAboutExportsDirectory) {
        warnedAboutExportsDirectory = true;
        console.info("EXPORT: Could not clean exports, directory", config.exportBase, "does not exist");
      }
      return;
    }
{'fraction_non_alphanumeric': 0.16400336417157274, 'fraction_numerical': 0.004906083543594056, 'mean_word_length': 2.348193336461755, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4296033', 'n_tokens_mistral': 2240, 'n_tokens_neox': 2194, 'n_words': 530}