<commit_before>'use strict';
const sysPath = require('universal-path');
const logger = require('loggy');
const debug = require('debug')('brunch:pipeline');
const deppack = require('deppack'); // isNpm
const BrunchError = require('../utils/error');

// Pipeline: File -> File.
// Takes file and a) lints b) compiles c) extracts dependencies from it.

const extRe = /(\.\w+)+$/;

let plugins = [];
let npmCompilers = [];
function respondTo(key) {
  return plugins.filter(plugin => {
    return typeof plugin[key] === 'function';
  });
}

function setPlugins(_plugins, _npmCompilers) {
  plugins = _plugins.all;
  npmCompilers = _npmCompilers;
  deppack.setPlugins(_plugins, npmCompilers);
}

function pull(array, predicate) {
  const index = array.findIndex(predicate);
  if (index === -1) return;

  const item = array[index];
  array.splice(index, 1);
  return item;
}

function rethrow(message) {
  return error => {
    if (!(error instanceof Error)) {
      error = new Error(error);
    }

    error.pipelineCode = message;
    throw error;
  };
}

async function lint(file) {
  const path = file.path;
  const linters = respondTo('lint').filter(l => l.pattern.test(path)).map(linter => {
    debug(`Linting ${path} @ ${linter.brunchPluginName}`);
    return linter.lint(file);
  });

  try {
    await Promise.all(linters);
  } catch (error) {
    if (/^warn:\s/i.test(error)) {
      logger.warn(`Linting of ${path}: ${error}`);
    } else {
      rethrow('Linting')(error);
    }
  }
  return file;
}

async function compileStatic(file, compiler) {
  const path = file.path;
  debug(`Compiling asset ${path} @ ${compiler.brunchPluginName}`);

  try {
    const compiled = await compiler.compileStatic(file);
    if (compiled == null) return file;
    if (compiled.data == null) {
      throw new BrunchError('FILE_DATA_INVALID', {path});
    }

    const dependencies = Array.isArray(compiled.dependencies) ?
      compiled.dependencies :
      await getDependencies(file, compiler);

    const cpath = compiled.path || file.path;
    return Object.assign({}, compiled, {
      dependencies,
      path: compiler.staticTargetExtension ?
        cpath.replace(extRe, compiler.staticTargetExtension) :
        cpath,
    });
  } catch (error) {
    rethrow('Compiling asset')(error);
  }
};

// Extract files' paths that depend on current file.
async function getDependencies(file, compiler) {
  if (typeof compiler.getDependencies !== 'function') {
    return Promise.resolve();
  }

  debug(`Fetching dependencies ${file.path} @ ${compiler.brunchPluginName}`);
  try {
    const deps = compiler.getDependencies(file);
    return deps.concat(deps.patterns || []).map(sysPath.normalize);
  } catch (error) {
    rethrow('Dependency parsing')(error);
  }
}

async function compile(file, compiler) {
  const path = file.path;
  debug(`Compiling ${path} @ ${compiler.brunchPluginName}`);

  try {
    const compiled = await compiler.compile(file);
    if (compiled == null) return file;
    if (compiled.data == null) {
      throw new BrunchError('FILE_DATA_INVALID', {path});
    }

    const dependencies = Array.isArray(compiled.dependencies) ?
      compiled.dependencies :
      await getDependencies(file, compiler);

    const cpath = compiled.path || file.path;
    return Object.assign({}, compiled, {
      dependencies: (file.dependencies || []).concat(dependencies || []),
      type: compiler.type,
      path: compiler.targetExtension ?
        cpath.replace(extRe, compiler.targetExtension) :
        cpath,
    });
  } catch (error) {
    rethrow('Compiling')(error);
  }
}

/**
 * @returns {Promise<File>}
 */
async function nextCompiler(compilers, isStatic, file) {
  const prop = isStatic ? 'staticPattern' : 'pattern';
  const exec = isStatic ? compileStatic : compile;
  const compiler = pull(compilers, comp => comp[prop].test(file.path));
  if (!compiler) return file;
  const compiled = await exec(file, compiler);
  return nextCompiler(compilers, isStatic, compiled);
}

async function processAsset(file) {
  const compilers = respondTo('compileStatic').filter(comp => comp.type === 'template');
  return nextCompiler(compilers, true, file);
}

async function processFile(file) {
  const {path} = file;
  const isNpm = deppack.isNpm(path);

  const usePlugin = isNpm ?
    compiler => npmCompilers.includes(compiler.brunchPluginName) :
    () => true;

  const lintedFile = await lint(file);
  const compilers = respondTo('compile').filter(usePlugin);
  const processed = await nextCompiler(compilers, false, lintedFile);
  if (!isNpm) return processed;

  const wrapped = await deppack.wrapSourceInModule(processed.data, sysPath.resolve(path));
  processed.data = wrapped;<commit_msg>Refactor pipeline more.<commit_after>'use strict';
const sysPath = require('universal-path');
const logger = require('loggy');
const debug = require('debug')('brunch:pipeline');
const deppack = require('deppack'); // isNpm
const BrunchError = require('../utils/error');

// Pipeline: File -> File.
// Takes file and a) lints b) compiles c) extracts dependencies from it.

const extRe = /(\.\w+)+$/;

let plugins = [];
let npmCompilers = [];
function respondTo(key) {
  return plugins.filter(plugin => {
    return typeof plugin[key] === 'function';
  });
}

function setPlugins(_plugins, _npmCompilers) {
  plugins = _plugins.all;
  npmCompilers = _npmCompilers;
  deppack.setPlugins(_plugins, npmCompilers);
}

function pull(array, predicate) {
  const index = array.findIndex(predicate);
  if (index === -1) return;

  const item = array[index];
  array.splice(index, 1);
  return item;
}

function rethrow(message) {
  return error => {
    if (!(error instanceof Error)) {
      error = new Error(error);
    }

    error.pipelineCode = message;
    throw error;
  };
}

async function lint(file) {
  const path = file.path;
  const linters = respondTo('lint').filter(l => l.pattern.test(path)).map(linter => {
    debug(`Linting ${path} @ ${linter.brunchPluginName}`);
    return linter.lint(file);
  });

  try {
    await Promise.all(linters);
    return true;
  } catch (error) {
    if (/^warn:\s/i.test(error)) {
      logger.warn(`Linting of ${path}: ${error}`);
    } else {
      rethrow('Linting')(error);
    }
    return false;
  }
}

// Extract files' paths that depend on current file.
async function getDependencies(file, compiler) {
  if (typeof compiler.getDependencies !== 'function') return [];

  debug(`Fetching dependencies ${file.path} @ ${compiler.brunchPluginName}`);
  try {
    const deps = compiler.getDependencies(file);
    return deps.concat(deps.patterns || []).map(sysPath.normalize);
  } catch (error) {
    rethrow('Dependency parsing')(error);
  }
}

async function compile(file, compiler, isStatic) {
  const {path} = file;
  const op = isStatic ? 'Compiling asset' : 'Compiling';
  debug(`${op} ${path} @ ${compiler.brunchPluginName}`);

  try {
    const method = isStatic ? 'compileStatic' : 'compile';
    const compiled = await compiler[method](file);
    if (compiled == null) return file;
    if (compiled.data == null) {
      throw new BrunchError('FILE_DATA_INVALID', {path});
    }
    const _deps = compiled.dependencies;
    const dependencies = Array.isArray(_deps) ? _deps : await getDependencies(file, compiler);
    const cpath = compiled.path || file.path;
    const additional = isStatic ? {
      dependencies,
      path: compiler.staticTargetExtension ?
        cpath.replace(extRe, compiler.staticTargetExtension) :
        cpath,
    } : {
      dependencies: (file.dependencies || []).concat(dependencies),
      type: compiler.type,
      path: compiler.targetExtension ?
        cpath.replace(extRe, compiler.targetExtension) :
        cpath,
    };

    return Object.assign({}, compiled, additional);
  } catch (error) {
    rethrow(op)(error);
  }
}

/**
 * @returns {Promise<File>}
 */
async function nextCompiler(file, compilers, isStatic) {
  const prop = isStatic ? 'staticPattern' : 'pattern';
  const compiler = pull(compilers, comp => comp[prop].test(file.path));
  if (!compiler) return file;
  const newFile = await compile(file, compiler, isStatic);
  return nextCompiler(newFile, compilers, isStatic);
}

async function processAsset(file) {
  const compilers = respondTo('compileStatic').filter(comp => comp.type === 'template');
  return nextCompiler(file, compilers, true);
}

async function processFile(file) {
  await lint(file);

  const {path} = file;
  const isNpm = deppack.isNpm(path);
  const usePlugin = isNpm ?
    comp => npmCompilers.includes(comp.brunchPluginName) :
    () => true;
  const compilers = respondTo('compile').filter(usePlugin);
  const processed = await nextCompiler(file, compilers, false);
  if (!isNpm) return processed;

  const wrapped = await deppack.wrapSourceInModule(processed.data, sysPath.resolve(path));
  processed.data = wrapped;
{'fraction_non_alphanumeric': 0.14636896707262065, 'fraction_numerical': 0.0004510599909788002, 'mean_word_length': 3.8650575973669774, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7214766', 'n_tokens_mistral': 2774, 'n_tokens_neox': 2716, 'n_words': 657}