<commit_before>/**
 * Main process entry point.
 *
 * This script runs boot scripts in order, wiring up Electron's main process and
 * kicking off the render process (UI).
 */

'use strict';

require('trace');
require('clarify');

Error.stackTraceLimit = 100;

const { compact } = require('lodash'); // eslint-disable-line no-unused-vars
const electron = require('electron');
const ipcPromise = require('ipc-promise');
const mock = require('../../test/e2e/mocks');
const ipcFunctions = require('./utils/ipc-functions');

const { ipcMain } = electron;

// if no env set prd
process.env.NODE_ENV = process.env.NODE_ENV || 'production';

// Mock file dialogue in testing environment
// Watch the following issue for progress on dialog support
// https://github.com/electron/spectron/issues/94
if (process.env.NODE_ENV === 'test') {
  mock(electron.dialog);
}

// Set up root paths
require('../common/utils/add-root-require-path.js');

// Parse and handle CLI flags
const parseCLIInput = require('./utils/boot/parse-cli-input.js');

parseCLIInput();

// Add dev mode specific services
require('./utils/boot/configure-dev-services.js');

// Load the UI
const getWindow = require('./utils/boot/configure-browser-window.js');

// Set up error handling
const logUnhandledError = require('../common/utils/log-unhandled-error.js');
const configureCore = require('./utils/boot/configure-core.js');
const configureLogger = require('./utils/boot/configure-logger.js');
const upsertCoinstacUserDir = require('./utils/boot/upsert-coinstac-user-dir.js');
const loadConfig = require('../config.js');
const fileFunctions = require('./services/files.js');

// Boot up the main process
loadConfig()
  .then(config => Promise.all([
    config,
    configureLogger(config),
  ]))
  .then(([config, logger]) => {
    process.on('uncaughtException', logUnhandledError(null, logger));
    global.config = config;

    const mainWindow = getWindow();
    let core = null;
    logger.verbose('main process booted');

    ipcMain.on('clean-remote-pipeline', (event, runId) => {
      if (core) {
        core.unlinkFiles(runId)
          .catch((err) => {
            logger.error(err);
            mainWindow.webContents.send('docker-error', {
              err: {
                message: err.message,
                stack: err.stack,
              },
            });
          });
      }
    });

    /**
   * IPC Listener to write logs
   * @param {String} message The message to write out to log
   * @param {String} type The type of log to write out
   */
    ipcMain.on('write-log', (event, { type, message }) => {
      logger[type](`process: render - ${message}`);
    });

    ipcPromise.on('login-init', (userId) => {
      return new Promise(res => res(configureCore(config, logger, userId)))
        .then((c) => {
          core = c;
          return upsertCoinstacUserDir(core);
        });
    });

    /**
   * IPC Listener to start pipeline
   * @param {Object} consortium
   * @param {String} consortium.id The id of the consortium starting the pipeline
   * @param {Object[]} consortium.pipelineSteps An array of the steps involved in
   *  this pipeline run according to the consortium
   * @param {String[]} filesArray An array of all the file locations used by this run
   * @param {Object} run
   * @param {String} run.id The id of the current run
   * @param {Object[]} run.pipelineSteps An array of the steps involved in this pipeline run
   *  according to the run
   * @return {Promise<String>} Status message
   */
    ipcMain.on('start-pipeline', (event, {
      consortium, pipeline, filesArray, run,
    }) => {
      const computationImageList = pipeline.steps
        .map(step => step.computations
          .map(comp => comp.computation.dockerImage))
        .reduce((acc, val) => acc.concat(val), []);

      return core.dockerManager.pullImagesFromList(computationImageList)
        .then((compStreams) => {
          const streamProms = [];

          compStreams.forEach(({ stream }) => {
            let proxRes;
            let proxRej;

            streamProms.push(new Promise((resolve, reject) => {
              proxRej = reject;
              proxRes = resolve;
            }));
            if (typeof stream.on !== 'function') {
              proxRej(stream.message);
            } else {
              mainWindow.webContents.send('local-pipeline-state-update', {
                run,
                data: { controllerState: 'Downloading required docker images' },
              });

              stream.on('data', (data) => {
                mainWindow.webContents.send('local-pipeline-state-update', {
                  run,
                  data: { controllerState: `Downloading required docker images\n ${data.toString()}` },
                });
              });

              stream.on('end', () => {
                proxRes();
              });

              stream.on('error', (err) => {
                proxRej(err);
              });
            }
          });

          return Promise.all(streamProms);
        })
        .catch((err) => {
          return core.unlinkFiles(run.id)
            .then(() => {
              mainWindow.webContents.send('local-run-error', {
                consName: consortium.name,
                run: Object.assign(
                  run,
                  {
                    error: {
                      message: err.message,
                      stack: err.stack,
                      error: err.error,
                    },
                    endDate: Date.now(),
                  }
                ),
              });
            });
        })
        .then(() => core.dockerManager.pruneImages())
        .then(() => {
          logger.verbose('############ CLIENT INPUT');
          logger.verbose(pipeline);
          logger.verbose('############ END CLIENT INPUT');
          return core.startPipeline(
            null,
            consortium.id,
            pipeline,
            filesArray,
            run.id,
            run.pipelineSteps
          )
            .then(([{ pipeline, result }]) => {
              // Listen for local pipeline state updates
              pipeline.stateEmitter.on('update', (data) => {
                mainWindow.webContents.send('local-pipeline-state-update', { run, data });
              });

              // Listen for results
              return result.then((results) => {
                logger.verbose('Pipeline is done. Result:'); // eslint-disable-line no-console
                logger.verbose(results); // eslint-disable-line no-console
                return core.unlinkFiles(run.id)
                  .then(() => {
                    if (run.type === 'local') {
                      mainWindow.webContents.send('local-run-complete', {
                        consName: consortium.name,
                        run: Object.assign(run, { results, endDate: Date.now() }),
                      });
                    }
                  });
              })
                .catch((error) => {
                  return core.unlinkFiles(run.id)
                    .then(() => {
                      mainWindow.webContents.send('local-run-error', {
                        consName: consortium.name,
                        run: Object.assign(
                          run,
                          {
                            error: {
                              message: error.message,
                              stack: error.stack,
                              error: error.error,
                              input: error.input,
                            },
                            endDate: Date.now(),
                          }
                        ),
                      });
                    });
                });
            });
        });
    });

    /**
  * IPC listener to return a list of all local Docker images
  * @return {Promise<String[]>} An array of all local Docker image names
  */
    ipcPromise.on('get-all-images', () => {
      return core.dockerManager.getImages()
        .then((data) => {
          return data;
        })
        .catch((err) => {
          logger.error(err);
          mainWindow.webContents.send('Can\'t list current images', {
            err: {
              message: err.message,
              stack: err.stack,
            },
          });
        });
    });


    /**
  * IPC listener to return status of Docker
  * @return {Promise<boolean[]>} Docker running?
  */
    ipcPromise.on('get-status', () => {
      return core.dockerManager.getStatus()
        .then((result) => {
          return result;
        })
        .catch((err) => {
          logger.error(err);
          mainWindow.webContents.send('Can\'t get current docker status', {
            err: {
              message: err.message,
              stack: err.stack,
            },
          });
        });
    });

    /**
  * IPC Listener to download a list of computations
  * @param {Object} params
  * @param {String[]} params.computations An array of docker image names
  * @param {String} params.consortiumId ID of the consortium, if relevant,
  *  associated with the computations being retrieved
  * @return {Promise}
  */
    ipcPromise.on('download-comps', (params) => { // eslint-disable-line no-unused-vars
      return core.dockerManager
        .pullImages(params.computations)
        .then((compStreams) => {
          let streamsComplete = 0;

          compStreams.forEach(({ compId, compName, stream }) => {
            if (typeof stream.on !== 'function') {
              const output = [{
                message: stream.message, status: 'error', statusCode: stream.statusCode, isErr: true,
              }];
              mainWindow.webContents.send('docker-out', { output, compId, compName });
            } else {
              stream.on('data', (data) => {
                let output = compact(data.toString().split('\r\n'));<commit_msg>Sends are not a message.....<commit_after>/**
 * Main process entry point.
 *
 * This script runs boot scripts in order, wiring up Electron's main process and
 * kicking off the render process (UI).
 */

'use strict';

require('trace');
require('clarify');

Error.stackTraceLimit = 100;

const { compact } = require('lodash'); // eslint-disable-line no-unused-vars
const electron = require('electron');
const ipcPromise = require('ipc-promise');
const mock = require('../../test/e2e/mocks');
const ipcFunctions = require('./utils/ipc-functions');

const { ipcMain } = electron;

// if no env set prd
process.env.NODE_ENV = process.env.NODE_ENV || 'production';

// Mock file dialogue in testing environment
// Watch the following issue for progress on dialog support
// https://github.com/electron/spectron/issues/94
if (process.env.NODE_ENV === 'test') {
  mock(electron.dialog);
}

// Set up root paths
require('../common/utils/add-root-require-path.js');

// Parse and handle CLI flags
const parseCLIInput = require('./utils/boot/parse-cli-input.js');

parseCLIInput();

// Add dev mode specific services
require('./utils/boot/configure-dev-services.js');

// Load the UI
const getWindow = require('./utils/boot/configure-browser-window.js');

// Set up error handling
const logUnhandledError = require('../common/utils/log-unhandled-error.js');
const configureCore = require('./utils/boot/configure-core.js');
const configureLogger = require('./utils/boot/configure-logger.js');
const upsertCoinstacUserDir = require('./utils/boot/upsert-coinstac-user-dir.js');
const loadConfig = require('../config.js');
const fileFunctions = require('./services/files.js');

// Boot up the main process
loadConfig()
  .then(config => Promise.all([
    config,
    configureLogger(config),
  ]))
  .then(([config, logger]) => {
    process.on('uncaughtException', logUnhandledError(null, logger));
    global.config = config;

    const mainWindow = getWindow();
    let core = null;
    logger.verbose('main process booted');

    ipcMain.on('clean-remote-pipeline', (event, runId) => {
      if (core) {
        core.unlinkFiles(runId)
          .catch((err) => {
            logger.error(err);
            mainWindow.webContents.send('docker-error', {
              err: {
                message: err.message,
                stack: err.stack,
              },
            });
          });
      }
    });

    /**
   * IPC Listener to write logs
   * @param {String} message The message to write out to log
   * @param {String} type The type of log to write out
   */
    ipcMain.on('write-log', (event, { type, message }) => {
      logger[type](`process: render - ${message}`);
    });

    ipcPromise.on('login-init', (userId) => {
      return new Promise(res => res(configureCore(config, logger, userId)))
        .then((c) => {
          core = c;
          return upsertCoinstacUserDir(core);
        });
    });

    /**
   * IPC Listener to start pipeline
   * @param {Object} consortium
   * @param {String} consortium.id The id of the consortium starting the pipeline
   * @param {Object[]} consortium.pipelineSteps An array of the steps involved in
   *  this pipeline run according to the consortium
   * @param {String[]} filesArray An array of all the file locations used by this run
   * @param {Object} run
   * @param {String} run.id The id of the current run
   * @param {Object[]} run.pipelineSteps An array of the steps involved in this pipeline run
   *  according to the run
   * @return {Promise<String>} Status message
   */
    ipcMain.on('start-pipeline', (event, {
      consortium, pipeline, filesArray, run,
    }) => {
      const computationImageList = pipeline.steps
        .map(step => step.computations
          .map(comp => comp.computation.dockerImage))
        .reduce((acc, val) => acc.concat(val), []);

      return core.dockerManager.pullImagesFromList(computationImageList)
        .then((compStreams) => {
          const streamProms = [];

          compStreams.forEach(({ stream }) => {
            let proxRes;
            let proxRej;

            streamProms.push(new Promise((resolve, reject) => {
              proxRej = reject;
              proxRes = resolve;
            }));
            if (typeof stream.on !== 'function') {
              proxRej(stream.message);
            } else {
              mainWindow.webContents.send('local-pipeline-state-update', {
                run,
                data: { controllerState: 'Downloading required docker images' },
              });

              stream.on('data', (data) => {
                mainWindow.webContents.send('local-pipeline-state-update', {
                  run,
                  data: { controllerState: `Downloading required docker images\n ${data.toString()}` },
                });
              });

              stream.on('end', () => {
                proxRes();
              });

              stream.on('error', (err) => {
                proxRej(err);
              });
            }
          });

          return Promise.all(streamProms);
        })
        .catch((err) => {
          return core.unlinkFiles(run.id)
            .then(() => {
              mainWindow.webContents.send('local-run-error', {
                consName: consortium.name,
                run: Object.assign(
                  run,
                  {
                    error: {
                      message: err.message,
                      stack: err.stack,
                      error: err.error,
                    },
                    endDate: Date.now(),
                  }
                ),
              });
            });
        })
        .then(() => core.dockerManager.pruneImages())
        .then(() => {
          logger.verbose('############ CLIENT INPUT');
          logger.verbose(pipeline);
          logger.verbose('############ END CLIENT INPUT');
          return core.startPipeline(
            null,
            consortium.id,
            pipeline,
            filesArray,
            run.id,
            run.pipelineSteps
          )
            .then(([{ pipeline, result }]) => {
              // Listen for local pipeline state updates
              pipeline.stateEmitter.on('update', (data) => {
                mainWindow.webContents.send('local-pipeline-state-update', { run, data });
              });

              // Listen for results
              return result.then((results) => {
                logger.verbose('Pipeline is done. Result:'); // eslint-disable-line no-console
                logger.verbose(results); // eslint-disable-line no-console
                return core.unlinkFiles(run.id)
                  .then(() => {
                    if (run.type === 'local') {
                      mainWindow.webContents.send('local-run-complete', {
                        consName: consortium.name,
                        run: Object.assign(run, { results, endDate: Date.now() }),
                      });
                    }
                  });
              })
                .catch((error) => {
                  return core.unlinkFiles(run.id)
                    .then(() => {
                      mainWindow.webContents.send('local-run-error', {
                        consName: consortium.name,
                        run: Object.assign(
                          run,
                          {
                            error: {
                              message: error.message,
                              stack: error.stack,
                              error: error.error,
                              input: error.input,
                            },
                            endDate: Date.now(),
                          }
                        ),
                      });
                    });
                });
            });
        });
    });

    /**
  * IPC listener to return a list of all local Docker images
  * @return {Promise<String[]>} An array of all local Docker image names
  */
    ipcPromise.on('get-all-images', () => {
      return core.dockerManager.getImages()
        .then((data) => {
          return data;
        })
        .catch((err) => {
          logger.error(err);
          mainWindow.webContents.send('docker-error', {
            err: {
              message: err.message,
              stack: err.stack,
            },
          });
        });
    });


    /**
  * IPC listener to return status of Docker
  * @return {Promise<boolean[]>} Docker running?
  */
    ipcPromise.on('get-status', () => {
      return core.dockerManager.getStatus()
        .then((result) => {
          return result;
        })
        .catch((err) => {
          logger.error(err);
          mainWindow.webContents.send('docker-error', {
            err: {
              message: err.message,
              stack: err.stack,
            },
          });
        });
    });

    /**
  * IPC Listener to download a list of computations
  * @param {Object} params
  * @param {String[]} params.computations An array of docker image names
  * @param {String} params.consortiumId ID of the consortium, if relevant,
  *  associated with the computations being retrieved
  * @return {Promise}
  */
    ipcPromise.on('download-comps', (params) => { // eslint-disable-line no-unused-vars
      return core.dockerManager
        .pullImages(params.computations)
        .then((compStreams) => {
          let streamsComplete = 0;

          compStreams.forEach(({ compId, compName, stream }) => {
            if (typeof stream.on !== 'function') {
              const output = [{
                message: stream.message, status: 'error', statusCode: stream.statusCode, isErr: true,
              }];
              mainWindow.webContents.send('docker-out', { output, compId, compName });
            } else {
              stream.on('data', (data) => {
                let output = compact(data.toString().split('\r\n'));
{'fraction_non_alphanumeric': 0.13635221390159322, 'fraction_numerical': 0.0007036236618585716, 'mean_word_length': 1.6456588219651642, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 85, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3417707', 'n_tokens_mistral': 5411, 'n_tokens_neox': 5201, 'n_words': 1204}