<commit_before>'use strict'

each = require 'async-each'
sysPath = require 'path'
generate = require './generate'
helpers = require '../helpers'

getPaths = (sourceFile, joinConfig) ->
  sourceFileJoinConfig = joinConfig[sourceFile.type + 's'] or {}
  Object.keys(sourceFileJoinConfig)
    .filter (key) ->
      key isnt 'pluginHelpers'
    .filter (generatedFilePath) ->
      if sourceFile.isHelper
        generatedFilePath in sourceFileJoinConfig.pluginHelpers
      else
        checker = sourceFileJoinConfig[generatedFilePath]
        checker sourceFile.path

getFiles = (fileList, config, joinConfig) ->
  map = {}

  fileList.files.forEach (file) ->
    return if not file.error? and not file.data?
    paths = getPaths file, joinConfig
    paths.forEach (path) ->
      map[path] ?= []
      map[path].push file

  Object.keys(map).map (generatedFilePath) ->
    sourceFiles = map[generatedFilePath]
    fullPath = sysPath.join config.paths.public, generatedFilePath
    {sourceFiles, path: fullPath}

changedSince = (startTime) -> (generated) ->
  generated.sourceFiles.some (sourceFile) ->
    sourceFile.compilationTime >= startTime or sourceFile.removed

formatError = (sourceFile) ->
  helpers.formatError sourceFile.error, sourceFile.path

module.exports = write = (fileList, config, joinConfig, optimizers, startTime, callback) ->
  files = getFiles fileList, config, joinConfig
  errors = files
    .map (generated) ->
      generated.sourceFiles.filter((_) -> _.error?).map(formatError)
    .reduce(((a, b) -> a.concat b), [])
  return callback errors.join(' ; ') if errors.length > 0  # callback errors

  changed = files.filter(changedSince startTime)

  # Remove files marked as such and dispose them, clean memory.
  disposed = generated: [], sourcePaths: []
  changed.forEach (generated) ->
    sourceFiles = generated.sourceFiles
    sourceFiles
      .filter (file) ->
        file.removed
      .forEach (file) ->
        disposed.generated.push generated
        disposed.sourcePaths.push sysPath.basename file.path
        file.dispose()
        sourceFiles.splice sourceFiles.indexOf(file), 1

  gen = (file, next) ->
    generate file.path, file.sourceFiles, config, optimizers, next
  each changed, gen, (error) ->
    return callback error if error?
    callback null, changed, disposed
<commit_msg>Print compiler errors for non-joined files<commit_after>'use strict'

each = require 'async-each'
sysPath = require 'path'
generate = require './generate'
helpers = require '../helpers'
logger = require 'loggy'

getPaths = (sourceFile, joinConfig) ->
  sourceFileJoinConfig = joinConfig[sourceFile.type + 's'] or {}
  Object.keys(sourceFileJoinConfig)
    .filter (key) ->
      key isnt 'pluginHelpers'
    .filter (generatedFilePath) ->
      if sourceFile.isHelper
        generatedFilePath in sourceFileJoinConfig.pluginHelpers
      else
        checker = sourceFileJoinConfig[generatedFilePath]
        checker sourceFile.path

getFiles = (fileList, config, joinConfig) ->
  map = {}

  fileList.files.forEach (file) ->
    return if not file.error? and not file.data?
    paths = getPaths file, joinConfig
    paths.forEach (path) ->
      map[path] ?= []
      map[path].push file
    unless paths.length
      if file.error
        logger.error formatError file

  Object.keys(map).map (generatedFilePath) ->
    sourceFiles = map[generatedFilePath]
    fullPath = sysPath.join config.paths.public, generatedFilePath
    {sourceFiles, path: fullPath}

changedSince = (startTime) -> (generated) ->
  generated.sourceFiles.some (sourceFile) ->
    sourceFile.compilationTime >= startTime or sourceFile.removed

formatError = (sourceFile) ->
  helpers.formatError sourceFile.error, sourceFile.path

module.exports = write = (fileList, config, joinConfig, optimizers, startTime, callback) ->
  files = getFiles fileList, config, joinConfig
  errors = files
    .map (generated) ->
      generated.sourceFiles.filter((_) -> _.error?).map(formatError)
    .reduce(((a, b) -> a.concat b), [])
  return callback errors.join(' ; ') if errors.length > 0  # callback errors

  changed = files.filter(changedSince startTime)

  # Remove files marked as such and dispose them, clean memory.
  disposed = generated: [], sourcePaths: []
  changed.forEach (generated) ->
    sourceFiles = generated.sourceFiles
    sourceFiles
      .filter (file) ->
        file.removed
      .forEach (file) ->
        disposed.generated.push generated
        disposed.sourcePaths.push sysPath.basename file.path
        file.dispose()
        sourceFiles.splice sourceFiles.indexOf(file), 1

  gen = (file, next) ->
    generate file.path, file.sourceFiles, config, optimizers, next
  each changed, gen, (error) ->
    return callback error if error?
    callback null, changed, disposed

{'fraction_non_alphanumeric': 0.11307788421491045, 'fraction_numerical': 0.0008329862557267805, 'mean_word_length': 3.3703366696997272, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5598415', 'n_tokens_mistral': 1369, 'n_tokens_neox': 1360, 'n_words': 365}