<commit_before>/**
 * Copyright (c) 2015-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 *
 * @flow
 */

'use strict';

const Logger = require('../Logger');

const declareOpts = require('../lib/declareOpts');
const denodeify = require('denodeify');
const os = require('os');
const util = require('util');
const workerFarm = require('worker-farm');
const debug = require('debug')('RNP:JStransformer');

import type {Data as TransformData, Options as TransformOptions} from './worker/worker';
import type {SourceMap} from '../lib/SourceMap';

// Avoid memory leaks caused in workers. This number seems to be a good enough number
// to avoid any memory leak while not slowing down initial builds.
// TODO(amasad): Once we get bundle splitting, we can drive this down a bit more.
const MAX_CALLS_PER_WORKER = 600;

// Worker will timeout if one of the callers timeout.
const TRANSFORM_TIMEOUT_INTERVAL = 301000;

// How may times can we tolerate failures from the worker.
const MAX_RETRIES = 2;

const validateOpts = declareOpts({
  transformModulePath: {
    type:'string',
    required: false,
  },
  worker: {
    type: 'string',
  },
  methods: {
    type: 'array',
    default: [],
  },
});

type Options = {
  transformModulePath?: ?string,
  worker?: ?string,
  methods?: ?Array<string>,
};

const maxConcurrentWorkers = ((cores, override) => {
  if (override) {
    return Math.min(cores, override);
  }

  if (cores < 3) {
    return cores;
  }
  if (cores < 8) {
    return Math.floor(cores * 0.75);
  }
  if (cores < 24) {
    return Math.floor(3 / 8 * cores + 3); // between cores *.75 and cores / 2
  }
  return cores / 2;
})(os.cpus().length, parseInt(process.env.REACT_NATIVE_MAX_WORKERS, 10));

function makeFarm(worker, methods, timeout) {
  return workerFarm(
    {
      autoStart: true,
      maxConcurrentCallsPerWorker: 1,
      maxConcurrentWorkers: maxConcurrentWorkers,
      maxCallsPerWorker: MAX_CALLS_PER_WORKER,
      maxCallTime: timeout,
      maxRetries: MAX_RETRIES,
    },
    worker,
    methods,
  );
}

class Transformer {

  _opts: {
    transformModulePath?: ?string,
    worker: ?string,
    methods: Array<string>,
  };
  _workers: {[name: string]: mixed};
  _transformModulePath: ?string;
  _transform: (
    transform: string,
    filename: string,
    sourceCode: string,
    options: ?TransformOptions,
  ) => Promise<TransformData>;
  minify: (
    filename: string,
    code: string,
    sourceMap: SourceMap,
  ) => Promise<{code: string, map: SourceMap}>;

  constructor(options: Options) {
    const opts = this._opts = validateOpts(options);

    const {transformModulePath} = opts;

    if (opts.worker) {
      this._workers =
        makeFarm(opts.worker, opts.methods, TRANSFORM_TIMEOUT_INTERVAL);
      opts.methods.forEach(name => {
        /* $FlowFixMe: assigning the class object fields directly is
         * questionable, because it's prone to conflicts. */
        this[name] = this._workers[name];
      });
    }
    else if (transformModulePath) {
      this._transformModulePath = require.resolve(transformModulePath);

      this._workers = makeFarm(
        require.resolve('./worker'),
        ['minify', 'transformAndExtractDependencies'],
        TRANSFORM_TIMEOUT_INTERVAL,
      );
      this._transform = denodeify(this._workers.transformAndExtractDependencies);
      this.minify = denodeify(this._workers.minify);
    }
  }

  kill() {
    this._workers && workerFarm.end(this._workers);
  }

  transformFile(fileName: string, code: string, options: TransformOptions) {
    if (!this._transform) {
      return Promise.reject(new Error('No transform module'));
    }
    debug('transforming file', fileName);
    return this
      /* $FlowFixMe: _transformModulePath may be empty, see constructor */
      ._transform(this._transformModulePath, fileName, code, options)
      .then(data => {
        Logger.log(data.transformFileStartLogEntry);
        Logger.log(data.transformFileEndLogEntry);
        debug('done transforming file', fileName);
        return data.result;
      })
      .catch(error => {<commit_msg>packager: JSTransform cleanup: remove opts.worker<commit_after>/**
 * Copyright (c) 2015-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 *
 * @flow
 */

'use strict';

const Logger = require('../Logger');

const declareOpts = require('../lib/declareOpts');
const denodeify = require('denodeify');
const os = require('os');
const util = require('util');
const workerFarm = require('worker-farm');
const debug = require('debug')('RNP:JStransformer');

import type {Data as TransformData, Options as TransformOptions} from './worker/worker';
import type {SourceMap} from '../lib/SourceMap';

// Avoid memory leaks caused in workers. This number seems to be a good enough number
// to avoid any memory leak while not slowing down initial builds.
// TODO(amasad): Once we get bundle splitting, we can drive this down a bit more.
const MAX_CALLS_PER_WORKER = 600;

// Worker will timeout if one of the callers timeout.
const TRANSFORM_TIMEOUT_INTERVAL = 301000;

// How may times can we tolerate failures from the worker.
const MAX_RETRIES = 2;

const validateOpts = declareOpts({
  transformModulePath: {
    type:'string',
    required: false,
  },
  methods: {
    type: 'array',
    default: [],
  },
});

type Options = {
  transformModulePath?: ?string,
  methods?: ?Array<string>,
};

const maxConcurrentWorkers = ((cores, override) => {
  if (override) {
    return Math.min(cores, override);
  }

  if (cores < 3) {
    return cores;
  }
  if (cores < 8) {
    return Math.floor(cores * 0.75);
  }
  if (cores < 24) {
    return Math.floor(3 / 8 * cores + 3); // between cores *.75 and cores / 2
  }
  return cores / 2;
})(os.cpus().length, parseInt(process.env.REACT_NATIVE_MAX_WORKERS, 10));

function makeFarm(worker, methods, timeout) {
  return workerFarm(
    {
      autoStart: true,
      maxConcurrentCallsPerWorker: 1,
      maxConcurrentWorkers: maxConcurrentWorkers,
      maxCallsPerWorker: MAX_CALLS_PER_WORKER,
      maxCallTime: timeout,
      maxRetries: MAX_RETRIES,
    },
    worker,
    methods,
  );
}

class Transformer {

  _opts: {
    transformModulePath?: ?string,
    methods: Array<string>,
  };
  _workers: {[name: string]: mixed};
  _transformModulePath: ?string;
  _transform: (
    transform: string,
    filename: string,
    sourceCode: string,
    options: ?TransformOptions,
  ) => Promise<TransformData>;
  minify: (
    filename: string,
    code: string,
    sourceMap: SourceMap,
  ) => Promise<{code: string, map: SourceMap}>;

  constructor(options: Options) {
    const opts = this._opts = validateOpts(options);

    const {transformModulePath} = opts;

    if (transformModulePath) {
      this._transformModulePath = require.resolve(transformModulePath);

      this._workers = makeFarm(
        require.resolve('./worker'),
        ['minify', 'transformAndExtractDependencies'],
        TRANSFORM_TIMEOUT_INTERVAL,
      );
      this._transform = denodeify(this._workers.transformAndExtractDependencies);
      this.minify = denodeify(this._workers.minify);
    }
  }

  kill() {
    this._workers && workerFarm.end(this._workers);
  }

  transformFile(fileName: string, code: string, options: TransformOptions) {
    if (!this._transform) {
      return Promise.reject(new Error('No transform module'));
    }
    debug('transforming file', fileName);
    return this
      /* $FlowFixMe: _transformModulePath may be empty, see constructor */
      ._transform(this._transformModulePath, fileName, code, options)
      .then(data => {
        Logger.log(data.transformFileStartLogEntry);
        Logger.log(data.transformFileEndLogEntry);
        debug('done transforming file', fileName);
        return data.result;
      })
      .catch(error => {
{'fraction_non_alphanumeric': 0.12058111380145278, 'fraction_numerical': 0.007506053268765133, 'mean_word_length': 3.4035181236673773, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4933551', 'n_tokens_mistral': 2557, 'n_tokens_neox': 2453, 'n_words': 697}