<commit_before>'use strict'

var isNode = (typeof window === 'undefined');

var path, slash, child_process;

var requireHack = require;

if (isNode) {
  path = requireHack("path");
  slash = requireHack("slash");
  child_process = requireHack("child_process");
}

function send(receiver, message) {
  if (isNode) {
    receiver.send(message);
  } else {
    receiver.postMessage(message);
  }
}

function createProcess(script, cwd) {
  if (isNode) {
    let p = child_process.fork(path.join(process.cwd(), script), {
      cwd: toAbsPath(cwd),
      // pipe stdout/stderr into this process
      silent: true
    });

    // handle stdout of child
    p.stdout.on("data", (data) => {
      process.stdout.write(`[Indexing] (${cwd}) ${data}`);
    });

    // handle stderr of child
    p.stderr.on("data", (data) => {
      process.stderr.write(`[Indexing] (${cwd}) ${data}`);
    });

    return p;
  }

  // in Browser:
  return new Worker(script);
}

// Object that holds workers for server subdirectories,
// e.g. '/lively4-core'
var workers = {};
var indexStatus = {};

var promiseCallbacks = {};
var workerReadyPromise;
// rootFolder is an absolute directory to where the server serves,
// e.g. where the lively4-core folder is found
var rootFolder = null;
var idxFileName = "index.l4idx";

var curMsgId = 0;


// *** Exported methods ***

export function setRootFolder(filepath) {
  // becomes e.g. "C:/felix/bla" on Windows
  if (isNode) {
    rootFolder = slash(path.resolve(filepath));
  } else {
    rootFolder = filepath;
  }
}

export function startWorker(subdir) {
  return new Promise((resolve, reject) => {
    if (!rootFolder) {
      throw new Error("startWorker: no root folder set");
    }

    if (workers[subdir]) {
      console.log("[Indexing] Worker already exists");
      resolve();
      return;
    }

    console.log("[Indexing] Starting new worker for " + subdir);
    try {
      let script = isNode ? "lunr-node-search-worker.js" : "../lively4-server/src/lunr-es6-search-worker-wrapper.js";
      workers[subdir] = createProcess(script, subdir);
    } catch (err) {
      console.log("[Indexing] Error starting new worker for " + subdir + ": " + err);
      reject(err);
      return;
    }

    let messageHandler = function(m) {
      // web workers hide the sent message behind the data key
      if (!isNode) {
        m = m.data;
      }
      switch (m.type) {
        case "worker-ready":
          resolve();
          break;
        case "search-response":
          handleSearchResponse(m.msgId, m.message);
          break;
        case "init-response":
          handleInitResponse(m.msgId, m.message, subdir);
          break;
        case "index-status-response":
          handleIndexStatusResponse(m.msgId, m.message);
          break;
        case "error":
          console.log("[Indexing] Error (" + subdir + "): " + m.message);
          break;
        default:
          console.log("[Indexing] Unknown message:", m);
      }
    }

    if (isNode) {
      workers[subdir].on("message", messageHandler);
    } else {
      workers[subdir].onmessage = messageHandler;
    }
  });
}

export function createIndex(subdir, options) {
  return new Promise((resolve, reject) => {
    if (!rootFolder) {
      console.log("[Indexing] Cannot create index, no root folder set");
      reject("Error: no root folder set");
      return;
    }

    if (!workers[subdir]) {
      console.log("[Indexing] Cannot create index, no worker running for ", subdir);
      reject("Error: no worker running");
      return;
    }

    if (indexStatus[subdir] === "ready") {
      console.log("[Indexing] Index already exists");
      resolve();
      return;
    }

    if (indexStatus[subdir] === "indexing") {
      reject();
      return;
    }

    var msgId = getNextMsgId();
    promiseCallbacks[msgId] = {
      resolve: resolve,
      reject: reject
    };

    send(workers[subdir], {
      type: "init",
      msgId: msgId,
      options: options
    });

    indexStatus[subdir] = "indexing";
  });
}
<commit_msg>refactor search module<commit_after>'use strict'

var isNode = (typeof window === 'undefined');

var path, slash, child_process;

var requireHack = require;

if (isNode) {
  path = requireHack("path");
  slash = requireHack("slash");
  child_process = requireHack("child_process");
}

function send(receiver, message) {
  if (isNode) {
    receiver.send(message);
  } else {
    receiver.postMessage(message);
  }
}

function createProcess(script, cwd) {
  if (isNode) {
    let p = child_process.fork(path.join(process.cwd(), script), {
      cwd: toAbsPath(cwd),
      // pipe stdout/stderr into this process
      silent: true
    });

    // handle stdout of child
    p.stdout.on("data", (data) => {
      process.stdout.write(`[Indexing] (${cwd}) ${data}`);
    });

    // handle stderr of child
    p.stderr.on("data", (data) => {
      process.stderr.write(`[Indexing] (${cwd}) ${data}`);
    });

    return p;
  }

  // in Browser:
  return new Worker(script);
}

// Object that holds workers for server subdirectories,
// e.g. '/lively4-core'
var workers = {};
var indexStatus = {};

var promiseCallbacks = {};
var workerReadyPromise;
// rootFolder is an absolute directory to where the server serves,
// e.g. where the lively4-core folder is found
var rootFolder = null;
var idxFileName = "index.l4idx";

var curMsgId = 0;


// *** Exported methods ***

export function setRootFolder(filepath) {
  // becomes e.g. "C:/felix/bla" on Windows
  if (isNode) {
    rootFolder = slash(path.resolve(filepath));
  } else {
    rootFolder = filepath;
  }
}

export function startWorker(subdir) {
  return new Promise((resolve, reject) => {
    if (!rootFolder) {
      throw new Error("startWorker: no root folder set");
    }

    if (workers[subdir]) {
      console.log("[Indexing] Worker already exists");
      resolve();
      return;
    }

    console.log("[Indexing] Starting new worker for " + subdir);
    try {
      let script = isNode ? "lunr-node-search-worker.js" : "../lively4-server/src/lunr-es6-search-worker-wrapper.js";
      workers[subdir] = createProcess(script, subdir);
    } catch (err) {
      console.log("[Indexing] Error starting new worker for " + subdir + ": " + err);
      reject(err);
      return;
    }

    let messageHandler = function(m) {
      // web workers hide the sent message behind the data key
      if (!isNode) {
        m = m.data;
      }
      switch (m.type) {
        case "worker-ready":
          resolve();
          break;
        case "search-response":
          handleSearchResponse(m.msgId, m.message);
          break;
        case "init-response":
          handleInitResponse(m.msgId, m.message, subdir);
          break;
        case "index-status-response":
          handleIndexStatusResponse(m.msgId, m.message);
          break;
        case "error":
          console.log("[Indexing] Error (" + subdir + "): " + m.message);
          break;
        default:
          console.log("[Indexing] Unknown message:", m);
      }
    }

    if (isNode) {
      workers[subdir].on("message", messageHandler);
    } else {
      workers[subdir].onmessage = messageHandler;
    }
  });
}

export function createIndex(subdir, options) {
  return startWorker(subdir).then( () => {
    return new Promise((resolve, reject) => {
      if (!rootFolder) {
        console.log("[Indexing] Cannot create index, no root folder set");
        reject("Error: no root folder set");
        return;
      }

      if (indexStatus[subdir] === "ready") {
        console.log("[Indexing] Index already exists");
        resolve();
        return;
      }

      if (indexStatus[subdir] === "indexing") {
        reject();
        return;
      }

      var msgId = getNextMsgId();
      promiseCallbacks[msgId] = {
        resolve: resolve,
        reject: reject
      };

      send(workers[subdir], {
        type: "init",
        msgId: msgId,
        options: options
      });

      indexStatus[subdir] = "indexing";
    });
  });
}

{'fraction_non_alphanumeric': 0.1463717693836978, 'fraction_numerical': 0.0014910536779324055, 'mean_word_length': 2.50718954248366, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2180211', 'n_tokens_mistral': 2539, 'n_tokens_neox': 2511, 'n_words': 617}