<commit_before>export let workspaces:{[name:string]: string} = {};

//---------------------------------------------------------
// Public
//---------------------------------------------------------

export function loadWorkspaces(neue) {
  for(let key in neue) {
    let value = neue[key];
    workspaces[key] = value;
  }
}

export function add(name: string, directory: string) {
  // If we're running on a windows server, normalize slashes
  if(typeof window === "undefined") {
    if(process.platform.indexOf("win") === 0) {
      directory = directory.replace(/\\/g, "/");
    }
  }

  if(directory[directory.length - 1] !== "/") directory += "/";

  if(workspaces[name] && workspaces[name] !== directory)
    throw new Error(`Unable to link pre-existing workspace '$[name}' to '${directory}' (currently '${workspaces[name]}')`);

  workspaces[name] = directory;
}

/** Given an explicit workspace, return the contents of the file. */
export function get(file:string, workspace = "eve"):string|undefined {
  if(!workspaces[workspace]) {
    console.error(`Unable to get '${file}' from unregistered workspace '${workspace}'`);
    return;
  }

  return fetchFile(file, workspace);
}

/** Using the inferred workspace from the file path, return the contents of the file. */
export function find(file:string):string|undefined {
  let workspace = getWorkspaceFromPath(file);
  if(!workspace) return;

  return get(file, workspace);
}

/** Given an explicit workspace, update the contents of the file. */
export function set(file:string, content:string, workspace = "eve") {
  if(!workspaces[workspace]) {
    console.error(`Unable to set '${file}' from unregistered workspace '${workspace}'`);
    return;
  }

  saveFile(file, content, workspace);
}

/** Using the inferred workspace from the file path, update the contents of the file. */
export function save(file:string, content:string) {
  let workspace = getWorkspaceFromPath(file);
  if(!workspace) return;

  return set(file, content, workspace);
}

//---------------------------------------------------------
// Utilities
//---------------------------------------------------------


export function getWorkspaceFromPath(file:string):string|undefined {
  var parts = file.split("/");
  var basename = parts.pop();
  var workspace = parts[1];
  if(!basename || !workspace) return;
  if(!workspaces[workspace]) {
    console.error(`Unable to get '${file}' from unregistered workspace '${workspace}'`);
  }

  return workspace;
}

export function getRelativePath(file:string, workspace:string):string|undefined {
  let directory = workspaces[workspace];
  if(!directory) {
    console.error(`Unable to get relative path for '${file}' in unregistered workspace '${workspace}'`);
    return;
  }

  if(file.indexOf("./") === 0) {
    file = file.slice(2);
  }

  if(file.indexOf(directory) === 0) {
    file = file.slice(directory.length);
  }
  return "/" + workspace + "/" + file;
}

export function getAbsolutePath(file:string, workspace:string) {
  let directory = workspaces[workspace];
  if(file.indexOf(directory) === 0) return file;

  if(file.indexOf("/" + workspace + "/") === 0) file = file.slice(workspace.length + 2);
  return directory + file;
}

//---------------------------------------------------------
// Server/Client Implementations
//---------------------------------------------------------

var saveFile = function(file:string, content:string, workspace:string) {
  let cache = global["_workspaceCache"][workspace];
  cache = global["_workspaceCache"][workspace] = {};
  file = getRelativePath(file, workspace);
  cache[file] = content;
}

// If we're running on the client, we use the global _workspaceCache, created in the build phase or served by the server.
var fetchFile = function(file:string, workspace:string):string|undefined {
  let cache = global["_workspaceCache"][workspace];
  file = getRelativePath(file, workspace);
  return cache && cache[file];
}

var fetchWorkspace = function(workspace:string) {
  return global["_workspaceCache"][workspace];
}

let fs = require("fs");
// If we're running on the server, we use the actual file-system.
if(fs.readFileSync) {
  let glob = require("glob");
  let path = require("path");
  let mkdirp = require("mkdirp");

  saveFile = function(file:string, content:string, workspace:string) {
    try {
      let filepath = getAbsolutePath(file, workspace);
      let dirname = path.dirname(filepath);
      mkdirp.sync(dirname);
      fs.writeFileSync(filepath, content);
    } catch(err) {
      console.warn(`Unable to save file '${file}' in '${workspace}' containing:\n${content}`);
    }
  }

  fetchFile = function(file:string, workspace:string):string|undefined {
    try {
      let filepath = getAbsolutePath(file, workspace);
      return fs.readFileSync(filepath).toString();
    } catch(err) {
      console.warn(`Unable to find file '${file}' in '${workspace}'`);
    }
  }

  fetchWorkspace = function(workspace:string) {
    let directory = workspaces[workspace];
    let files = {};
    let patterns = ["/**/*.eve", "/**/*.eve.md"];
    for(let pattern of patterns) {
      for(let file of glob.sync(directory + pattern, {ignore: directory + "**/node_modules" + pattern})) {
        let rel = path.relative(directory, file);
        files["/" + workspace + "/" + rel] = fs.readFileSync(file).toString();
      }
    }

    return files;
  }
}

export function pack() {
  let packaged = {};
  for(let workspace in workspaces) {
    packaged[workspace] = fetchWorkspace(workspace);
  }

  return `var _workspaceCache = ${JSON.stringify(packaged, null, 2)};\n`;
}

// If we're running on the client, load the server's workspaces from the cache it passes us.
if(global["_workspaceCache"]) {
  for(let workspace in global["_workspaceCache"]) {
    add(workspace, workspace);
  }
}<commit_msg>Add fetchAll to grab all docs in workspace as an obj<commit_after>export let workspaces:{[name:string]: string} = {};

//---------------------------------------------------------
// Public
//---------------------------------------------------------

export function loadWorkspaces(neue) {
  for(let key in neue) {
    let value = neue[key];
    workspaces[key] = value;
  }
}

export function add(name: string, directory: string) {
  // If we're running on a windows server, normalize slashes
  if(typeof window === "undefined") {
    if(process.platform.indexOf("win") === 0) {
      directory = directory.replace(/\\/g, "/");
    }
  }

  if(directory[directory.length - 1] !== "/") directory += "/";

  if(workspaces[name] && workspaces[name] !== directory)
    throw new Error(`Unable to link pre-existing workspace '$[name}' to '${directory}' (currently '${workspaces[name]}')`);

  workspaces[name] = directory;
}

/** Given an explicit workspace, return the contents of the file. */
export function get(file:string, workspace = "eve"):string|undefined {
  if(!workspaces[workspace]) {
    console.error(`Unable to get '${file}' from unregistered workspace '${workspace}'`);
    return;
  }

  return fetchFile(file, workspace);
}

/** Using the inferred workspace from the file path, return the contents of the file. */
export function find(file:string):string|undefined {
  let workspace = getWorkspaceFromPath(file);
  if(!workspace) return;

  return get(file, workspace);
}

/** Given an explicit workspace, update the contents of the file. */
export function set(file:string, content:string, workspace = "eve") {
  if(!workspaces[workspace]) {
    console.error(`Unable to set '${file}' from unregistered workspace '${workspace}'`);
    return;
  }

  saveFile(file, content, workspace);
}

/** Using the inferred workspace from the file path, update the contents of the file. */
export function save(file:string, content:string) {
  let workspace = getWorkspaceFromPath(file);
  if(!workspace) return;

  return set(file, content, workspace);
}

//---------------------------------------------------------
// Utilities
//---------------------------------------------------------


export function getWorkspaceFromPath(file:string):string|undefined {
  var parts = file.split("/");
  var basename = parts.pop();
  var workspace = parts[1];
  if(!basename || !workspace) return;
  if(!workspaces[workspace]) {
    console.error(`Unable to get '${file}' from unregistered workspace '${workspace}'`);
  }

  return workspace;
}

export function getRelativePath(file:string, workspace:string):string|undefined {
  let directory = workspaces[workspace];
  if(!directory) {
    console.error(`Unable to get relative path for '${file}' in unregistered workspace '${workspace}'`);
    return;
  }

  if(file.indexOf("./") === 0) {
    file = file.slice(2);
  }

  if(file.indexOf(directory) === 0) {
    file = file.slice(directory.length);
  }
  return "/" + workspace + "/" + file;
}

export function getAbsolutePath(file:string, workspace:string) {
  let directory = workspaces[workspace];
  if(file.indexOf(directory) === 0) return file;

  if(file.indexOf("/" + workspace + "/") === 0) file = file.slice(workspace.length + 2);
  return directory + file;
}

//---------------------------------------------------------
// Server/Client Implementations
//---------------------------------------------------------

var saveFile = function(file:string, content:string, workspace:string) {
  let cache = global["_workspaceCache"][workspace];
  cache = global["_workspaceCache"][workspace] = {};
  file = getRelativePath(file, workspace);
  cache[file] = content;
}

// If we're running on the client, we use the global _workspaceCache, created in the build phase or served by the server.
var fetchFile = function(file:string, workspace:string):string|undefined {
  let cache = global["_workspaceCache"][workspace];
  file = getRelativePath(file, workspace);
  return cache && cache[file];
}

var fetchWorkspace = function(workspace:string) {
  return global["_workspaceCache"][workspace];
}

let fs = require("fs");
// If we're running on the server, we use the actual file-system.
if(fs.readFileSync) {
  let glob = require("glob");
  let path = require("path");
  let mkdirp = require("mkdirp");

  saveFile = function(file:string, content:string, workspace:string) {
    try {
      let filepath = getAbsolutePath(file, workspace);
      let dirname = path.dirname(filepath);
      mkdirp.sync(dirname);
      fs.writeFileSync(filepath, content);
    } catch(err) {
      console.warn(`Unable to save file '${file}' in '${workspace}' containing:\n${content}`);
    }
  }

  fetchFile = function(file:string, workspace:string):string|undefined {
    try {
      let filepath = getAbsolutePath(file, workspace);
      return fs.readFileSync(filepath).toString();
    } catch(err) {
      console.warn(`Unable to find file '${file}' in '${workspace}'`);
    }
  }

  fetchWorkspace = function(workspace:string) {
    let directory = workspaces[workspace];
    let files = {};
    let patterns = ["/**/*.eve", "/**/*.eve.md"];
    for(let pattern of patterns) {
      for(let file of glob.sync(directory + pattern, {ignore: directory + "**/node_modules" + pattern})) {
        let rel = path.relative(directory, file);
        files["/" + workspace + "/" + rel] = fs.readFileSync(file).toString();
      }
    }

    return files;
  }
}

export function pack() {
  let packaged = {};
  for(let workspace in workspaces) {
    packaged[workspace] = fetchWorkspace(workspace);
  }

  return `var _workspaceCache = ${JSON.stringify(packaged, null, 2)};\n`;
}

export function fetchAll(workspace:string) {
  return fetchWorkspace(workspace);
}

// If we're running on the client, load the server's workspaces from the cache it passes us.
if(global["_workspaceCache"]) {
  for(let workspace in global["_workspaceCache"]) {
    add(workspace, workspace);
  }
}
{'fraction_non_alphanumeric': 0.2015780096716722, 'fraction_numerical': 0.0016967845931958937, 'mean_word_length': 3.1434094903339194, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2232841', 'n_tokens_mistral': 3427, 'n_tokens_neox': 3347, 'n_words': 899}