<commit_before>/*
 * Copyright 2009 The Closure Compiler Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.javascript.jscomp;

import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.javascript.jscomp.CompilerOptions.TweakProcessing;
import com.google.javascript.jscomp.deps.ClosureBundler;
import com.google.javascript.jscomp.deps.SourceCodeEscapers;
import com.google.javascript.rhino.Node;
import com.google.javascript.rhino.TokenStream;
import com.google.protobuf.CodedOutputStream;

import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.Flushable;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.StringWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;

import javax.annotation.Nullable;

/**
 * Implementations of AbstractCommandLineRunner translate flags into Java
 * API calls on the Compiler. AbstractCompiler contains common flags and logic
 * to make that happen.
 *
 * This class may be extended and used to create other Java classes
 * that behave the same as running the Compiler from the command line. Example:
 *
 * <pre>
 * class MyCommandLineRunner extends
 *     AbstractCommandLineRunner<MyCompiler, MyOptions> {
 *   MyCommandLineRunner(String[] args) {
 *     super(args);
 *   }
 *
 *   &#064;Override
 *   protected MyOptions createOptions() {
 *     MyOptions options = new MyOptions();
 *     CompilerFlagTranslator.setOptionsFromFlags(options);
 *     addMyCrazyCompilerPassThatOutputsAnExtraFile(options);
 *     return options;
 *   }
 *
 *   &#064;Override
 *   protected MyCompiler createCompiler() {
 *     return new MyCompiler();
 *   }
 *
 *   public static void main(String[] args) {
 *     (new MyCommandLineRunner(args)).run();
 *   }
 * }
 * </pre>
 *
 * @author <EMAIL> (<NAME>)
 */
abstract class AbstractCommandLineRunner<A extends Compiler,
    B extends CompilerOptions> {
  static final DiagnosticType OUTPUT_SAME_AS_INPUT_ERROR = DiagnosticType.error(
      "JSC_OUTPUT_SAME_AS_INPUT_ERROR",
      "Bad output file (already listed as input file): {0}");
  static final DiagnosticType NO_TREE_GENERATED_ERROR = DiagnosticType.error(
      "JSC_NO_TREE_GENERATED_ERROR",
      "Code contains errors. No tree was generated.");

  private final CommandLineConfig config;

  private final PrintStream defaultJsOutput;
  private final PrintStream err;
  private A compiler;

  private Charset inputCharset;

  // NOTE(nicksantos): JSCompiler has always used ASCII as the default
  // output charset. This was done to solve legacy problems with
  // bad proxies, etc. We are not sure if these issues are still problems,
  // and changing the encoding would require a big obnoxious migration plan.
  //
  // New outputs should use outputCharset2, which is how I would have
  // designed this if I had a time machine.
  private Charset outputCharset2;
  private String legacyOutputCharset;

  private boolean testMode = false;
  private Supplier<List<SourceFile>> externsSupplierForTesting = null;
  private Supplier<List<SourceFile>> inputsSupplierForTesting = null;
  private Supplier<List<JSModule>> modulesSupplierForTesting = null;
  private Function<Integer, Boolean> exitCodeReceiverForTesting = null;
  private Map<String, String> rootRelativePathsMap = null;

  private Map<String, String> parsedModuleWrappers = null;

  static final String OUTPUT_MARKER = "%output%";
  private static final String OUTPUT_MARKER_JS_STRING = "%output|jsstring%";

  AbstractCommandLineRunner() {
    this(System.out, System.err);
  }

  AbstractCommandLineRunner(PrintStream out, PrintStream err) {
    this.config = new CommandLineConfig();
    this.defaultJsOutput = Preconditions.checkNotNull(out);
    this.err = Preconditions.checkNotNull(err);
  }

  /**
   * Put the command line runner into test mode. In test mode,
   * all outputs will be blackholed.
   * @param externsSupplier A provider for externs.
   * @param inputsSupplier A provider for source inputs.
   * @param modulesSupplier A provider for modules. Only one of inputsSupplier
   *     and modulesSupplier may be non-null.
   * @param exitCodeReceiver A receiver for the status code that would
   *     have been passed to System.exit in non-test mode.
   */
  @VisibleForTesting
  void enableTestMode(
      Supplier<List<SourceFile>> externsSupplier,
      Supplier<List<SourceFile>> inputsSupplier,
      Supplier<List<JSModule>> modulesSupplier,
      Function<Integer, Boolean> exitCodeReceiver) {
    Preconditions.checkArgument(
        inputsSupplier == null ^ modulesSupplier == null);
    testMode = true;
    this.externsSupplierForTesting = externsSupplier;
    this.inputsSupplierForTesting = inputsSupplier;
    this.modulesSupplierForTesting = modulesSupplier;
    this.exitCodeReceiverForTesting = exitCodeReceiver;
  }

  /**
   * Returns whether we're in test mode.
   */
  protected boolean isInTestMode() {
    return testMode;
  }

  /**
   * Get the command line config, so that it can be initialized.
   */
  protected CommandLineConfig getCommandLineConfig() {
    return config;
  }

  /**
   * Returns the instance of the Compiler to use when {@link #run()} is
   * called.
   */
  protected abstract A createCompiler();

  /**
   * Returns the instance of the Options to use when {@link #run()} is called.
   * createCompiler() is called before createOptions(), so getCompiler()
   * will not return null when createOptions() is called.
   */
  protected abstract B createOptions();

  /**
   * The warning classes that are available from the command-line.
   */
  protected DiagnosticGroups getDiagnosticGroups() {
    if (compiler == null) {
      return new DiagnosticGroups();
    }
    return compiler.getDiagnosticGroups();
  }

  /**
   * A helper function for creating the dependency options object.
   */
  static DependencyOptions createDependencyOptions(
      boolean manageClosureDependencies,
      boolean onlyClosureDependencies,
      boolean processCommonJSModules,
      boolean rewriteEs6Modules,
      List<String> closureEntryPoints)
      throws FlagUsageException {
    if (onlyClosureDependencies) {
      if (closureEntryPoints.isEmpty()) {
        throw new FlagUsageException("When only_closure_dependencies is "
          + "on, you must specify at least one closure_entry_point");
      }

      return new DependencyOptions()
          .setDependencyPruning(true)
          .setDependencySorting(true)
          .setMoocherDropping(true)
          .setEntryPoints(closureEntryPoints);
    } else if (processCommonJSModules) {
      return new DependencyOptions()
        .setDependencyPruning(false)
        .setDependencySorting(true)
        .setMoocherDropping(false)
        .setEntryPoints(closureEntryPoints);
    } else if (rewriteEs6Modules) {
      return new DependencyOptions()
        .setDependencyPruning(false)
        .setDependencySorting(true)
        .setMoocherDropping(false)
        .setEntryPoints(closureEntryPoints);
    } else if (manageClosureDependencies || !closureEntryPoints.isEmpty()) {
      return new DependencyOptions()
          .setDependencyPruning(true)
          .setDependencySorting(true)
          .setMoocherDropping(false)
          .setEntryPoints(closureEntryPoints);
    }
    return null;
  }

  protected abstract void addWhitelistWarningsGuard(
      CompilerOptions options, File whitelistFile);

  /**
   * Sets options based on the configurations set flags API.
   * Called during the run() run() method.
   * If you want to ignore the flags API, or interpret flags your own way,
   * then you should override this method.
   */
  protected void setRunOptions(CompilerOptions options)
      throws FlagUsageException, IOException {
    DiagnosticGroups diagnosticGroups = getDiagnosticGroups();

    if (config.warningGuards != null) {
      for (WarningGuardSpec.Entry entry : config.warningGuards.entries) {
        diagnosticGroups.setWarningLevel(options, entry.groupName, entry.level);
      }
    }

    if (!config.warningsWhitelistFile.isEmpty()) {
      addWhitelistWarningsGuard(options, new File(config.warningsWhitelistFile));
    }

    createDefineOrTweakReplacements(config.define, options, false);

    options.setTweakProcessing(config.tweakProcessing);
    createDefineOrTweakReplacements(config.tweak, options, true);

    DependencyOptions depOptions = createDependencyOptions(
        config.manageClosureDependencies,
        config.onlyClosureDependencies,
        config.processCommonJSModules,
        config.rewriteEs6Modules,
        config.closureEntryPoints);
    if (depOptions != null) {
      options.setDependencyOptions(depOptions);
    }

    options.devMode = config.jscompDevMode;
    options.setCodingConvention(config.codingConvention);
    options.setSummaryDetailLevel(config.summaryDetailLevel);
    options.setTrustedStrings(true);

    legacyOutputCharset = options.outputCharset = getLegacyOutputCharset();
    outputCharset2 = getOutputCharset2();
    inputCharset = getInputCharset();

    if (config.jsOutputFile.length() > 0) {
      if (config.skipNormalOutputs) {
        throw new FlagUsageException("skip_normal_outputs and js_output_file"
            + " cannot be used together.");
      }
    }

    if (config.skipNormalOutputs && config.printAst) {
      throw new FlagUsageException("skip_normal_outputs and print_ast cannot"
          + " be used together.");
    }

    if (config.skipNormalOutputs && config.printTree) {
      throw new FlagUsageException("skip_normal_outputs and print_tree cannot"
          + " be used together.");
    }

    if (config.createSourceMap.length() > 0) {
      options.sourceMapOutputPath = config.createSourceMap;
    }
    options.sourceMapDetailLevel = config.sourceMapDetailLevel;
    options.sourceMapFormat = config.sourceMapFormat;
    options.sourceMapLocationMappings = config.sourceMapLocationMappings;

    ImmutableMap.Builder<String, SourceMapInput> inputSourceMaps
        = new ImmutableMap.Builder<>();
    for (Map.Entry<String, String> files :
             config.sourceMapInputFiles.entrySet()) {
      SourceFile sourceMap = SourceFile.fromFile(files.getValue());
      inputSourceMaps.put(
          files.getKey(), new SourceMapInput(sourceMap));
    }
    options.inputSourceMaps = inputSourceMaps.build();

    if (!config.variableMapInputFile.isEmpty()) {
      options.inputVariableMap =
          VariableMap.load(config.variableMapInputFile);
    }

    if (!config.propertyMapInputFile.isEmpty()) {
      options.inputPropertyMap =
          VariableMap.load(config.propertyMapInputFile);
    }

    if (config.languageIn.length() > 0) {
      CompilerOptions.LanguageMode languageMode =
          CompilerOptions.LanguageMode.fromString(config.languageIn);
      if (languageMode != null) {
        options.setLanguageIn(languageMode);
      } else {
        throw new FlagUsageException("Unknown language `" + config.languageIn +
                                     "' specified.");
      }
    }

    if (config.languageOut.isEmpty()) {
      options.setLanguageOut(options.getLanguageIn());
    } else {
      CompilerOptions.LanguageMode languageMode =
          CompilerOptions.LanguageMode.fromString(config.languageOut);
      if (languageMode != null) {
        options.setLanguageOut(languageMode);
      } else {
        throw new FlagUsageException("Unknown language `" + config.languageOut +
                                     "' specified.");
      }
    }

    if (!config.outputManifests.isEmpty()) {
      Set<String> uniqueNames = Sets.newHashSet();
      for (String filename : config.outputManifests) {
        if (!uniqueNames.add(filename)) {
          throw new FlagUsageException("output_manifest flags specify " +
              "duplicate file names: " + filename);
        }
      }
    }

    if (!config.outputBundles.isEmpty()) {
      Set<String> uniqueNames = Sets.newHashSet();
      for (String filename : config.outputBundles) {
        if (!uniqueNames.add(filename)) {
          throw new FlagUsageException("output_bundle flags specify " +
              "duplicate file names: " + filename);
        }
      }
    }

    options.acceptConstKeyword = config.acceptConstKeyword;
    options.transformAMDToCJSModules = config.transformAMDToCJSModules;
    options.processCommonJSModules = config.processCommonJSModules;
    options.rewriteEs6Modules = config.rewriteEs6Modules;
    options.transpileOnly = config.transpileOnly;
    options.commonJSModulePathPrefix = config.commonJSModulePathPrefix;
    options.angularPass = config.angularPass;
    options.tracer = config.tracerMode;
    options.useNewTypeInference = config.useNewTypeInference;
  }

  protected final A getCompiler() {
    return compiler;
  }

  /**
   * Runs the Compiler and calls System.exit() with the exit status of the
   * compiler.
   */
  public final void run() {
    int result = 0;
    int runs = 1;
    try {
      for (int i = 0; i < runs && result == 0; i++) {
        result = doRun();
      }
    } catch (AbstractCommandLineRunner.FlagUsageException e) {
      System.err.println(e.getMessage());
      result = -1;
    } catch (Throwable t) {
      t.printStackTrace();
      result = -2;
    }

    if (testMode) {
      exitCodeReceiverForTesting.apply(result);
    } else {
      System.exit(result);
    }
  }

  /**
   * Returns the PrintStream for writing errors associated with this
   * AbstractCommandLineRunner.
   */
  protected PrintStream getErrorPrintStream() {
    return err;
  }

  /**
   * An exception thrown when command-line flags are used incorrectly.
   */
  public static class FlagUsageException extends Exception {
    private static final long serialVersionUID = 1L;

    public FlagUsageException(String message) {
      super(message);
    }
  }

  /**
   * Creates inputs from a list of files.
   *
   * Can be overridden by subclasses who want to pull files from different
   * places.
   *
   * @param files A list of filenames
   * @param allowStdIn Whether '-' is allowed appear as a filename to represent
   *        stdin. If true, '-' is only allowed to appear once.
   * @return An array of inputs
   */
  protected List<SourceFile> createInputs(List<String> files,
      boolean allowStdIn) throws FlagUsageException, IOException {
    List<SourceFile> inputs = new ArrayList<>(files.size());
    boolean usingStdin = false;
    for (String filename : files) {
      if (!"-".equals(filename)) {
        SourceFile newFile = SourceFile.fromFile(filename, inputCharset);
        inputs.add(newFile);
      } else {
        if (!allowStdIn) {
          throw new FlagUsageException("Can't specify stdin.");
        }
        if (usingStdin) {
          throw new FlagUsageException("Can't specify stdin twice.");
        }

        if (!config.outputManifests.isEmpty()) {
          throw new FlagUsageException("Manifest files cannot be generated " +
              "when the input is from stdin.");
        }
        if (!config.outputBundles.isEmpty()) {
          throw new FlagUsageException("Bundle files cannot be generated " +
              "when the input is from stdin.");
        }
        inputs.add(SourceFile.fromInputStream("stdin", System.in, inputCharset));
        usingStdin = true;
      }
    }
    return inputs;
  }

  /**
   * Creates JS source code inputs from a list of files.
   */
  private List<SourceFile> createSourceInputs(List<String> files)
      throws FlagUsageException, IOException {
    if (isInTestMode()) {
      return inputsSupplierForTesting.get();
    }
    if (files.isEmpty()) {
      files = Collections.singletonList("-");
    }
    try {
      return createInputs(files, true);
    } catch (FlagUsageException e) {
      throw new FlagUsageException("Bad --js flag. " + e.getMessage());
    }
  }

  /**
   * Creates JS extern inputs from a list of files.
   */
  private List<SourceFile> createExternInputs(List<String> files)
      throws FlagUsageException, IOException {
    if (files.isEmpty()) {
      return ImmutableList.of(SourceFile.fromCode("/dev/null", ""));
    }
    try {
      return createInputs(files, false);
    } catch (FlagUsageException e) {
      throw new FlagUsageException("Bad --externs flag. " + e.getMessage());
    }
  }

  /**
   * Creates module objects from a list of module specifications.
   *
   * @param specs A list of module specifications, not null or empty. The spec
   *        format is: <code>name:num-js-files[:[dep,...][:]]</code>. Module
   *        names must not contain the ':' character.
   * @param jsFiles A list of JS file paths, not null
   * @return An array of module objects
   */
  List<JSModule> createJsModules(
      List<String> specs, List<String> jsFiles)
      throws FlagUsageException, IOException {
    if (isInTestMode()) {
      return modulesSupplierForTesting.get();
    }

    Preconditions.checkState(specs != null);
    Preconditions.checkState(!specs.isEmpty());
    Preconditions.checkState(jsFiles != null);

    List<String> moduleNames = new ArrayList<>(specs.size()); 
    Map<String, JSModule> modulesByName = Maps.newLinkedHashMap();
    Map<String, Integer> modulesFileCountMap = Maps.newLinkedHashMap();
    int numJsFilesExpected = 0, minJsFilesRequired = 0;
    boolean isFirstModule = true;
    for (String spec : specs) {
      // Format is "<name>:<num-js-files>[:[<dep>,...][:]]".
      String[] parts = spec.split(":");
      if (parts.length < 2 || parts.length > 4) {
        throw new FlagUsageException("Expected 2-4 colon-delimited parts in "
            + "module spec: " + spec);
      }

      // Parse module name.
      String name = parts[0];
      checkModuleName(name);
      if (modulesByName.containsKey(name)) {
        throw new FlagUsageException("Duplicate module name: " + name);
      }
      JSModule module = new JSModule(name);
      
      if (parts.length > 2) {
        // Parse module dependencies.
        String depList = parts[2];
        if (depList.length() > 0) {
          String[] deps = depList.split(",");
          for (String dep : deps) {
            JSModule other = modulesByName.get(dep);
            if (other == null) {
              throw new FlagUsageException("Module '" + name
                  + "' depends on unknown module '" + dep
                  + "'. Be sure to list modules in dependency order.");
            }
            module.addDependency(other);
          }
        }
      }
      
      // Parse module inputs.
      int numJsFiles = -1;
      try {
        numJsFiles = Integer.parseInt(parts[1]);
      } catch (NumberFormatException ignored) {
        numJsFiles = -1;
      }

      // We will allow modules of zero input.
      if (numJsFiles < 0) {
        // A size of 'auto' is only allowed on the base module if
        // and it must also be the first module
        if (parts.length == 2 && "auto".equals(parts[1])) {
          if (isFirstModule) {
            numJsFilesExpected = -1;
          } else {
            throw new FlagUsageException("Invalid JS file count '" + parts[1]
                + "' for module: " + name + ". Only the first module may specify " +
                "a size of 'auto' and it must have no dependencies.");
          }
        } else {
          throw new FlagUsageException("Invalid JS file count '" + parts[1]
              + "' for module: " + name);
        }
      } else {
        minJsFilesRequired += numJsFiles;
      }
      
      
      if (numJsFilesExpected >= 0) {
        numJsFilesExpected += numJsFiles;
      }
      
      // Add modules in reverse order so that source files are allocated to
      // modules in reverse order. This allows the first module
      // (presumably the base module) to have a size of 'auto'
      moduleNames.add(0, name);
      modulesFileCountMap.put(name, numJsFiles);
      modulesByName.put(name, module);
    }
        
    final int totalNumJsFiles = jsFiles.size();
    
    if (numJsFilesExpected >= 0 || minJsFilesRequired > totalNumJsFiles) {
      if (minJsFilesRequired > totalNumJsFiles) {
        numJsFilesExpected = minJsFilesRequired;
      }
      
      if (numJsFilesExpected > totalNumJsFiles) {        
        throw new FlagUsageException("Not enough JS files specified. Expected "
            + numJsFilesExpected + " but found " + totalNumJsFiles);
      } else if (numJsFilesExpected < totalNumJsFiles) {
        throw new FlagUsageException("Too many JS files specified. Expected "
                + numJsFilesExpected + " but found " + totalNumJsFiles);
      }
    }
    
    int numJsFilesLeft = totalNumJsFiles, moduleIndex = 0;
    for(String moduleName : moduleNames) {
      // Parse module inputs.
      int numJsFiles = modulesFileCountMap.get(moduleName);
      JSModule module = modulesByName.get(moduleName);

      // Check if the first module specified 'auto' for the number of files
      if (moduleIndex == moduleNames.size() - 1 && numJsFiles == -1) {
        numJsFiles = numJsFilesLeft;
      }
      
      List<String> moduleJsFiles =
          jsFiles.subList(numJsFilesLeft - numJsFiles, numJsFilesLeft);
      for (SourceFile input : createInputs(moduleJsFiles, false)) {
        module.add(input);
      }
      numJsFilesLeft -= numJsFiles;
      moduleIndex++;
    }

    return Lists.newArrayList(modulesByName.values());
  }

  /**
   * Validates the module name. Can be overridden by subclasses.
   * @param name The module name<commit_msg>Alphabetize JavaScript extern declarations for Android GSA. ------------- Created by MOE: http://code.google.com/p/moe-java MOE_MIGRATED_REVID=82479459<commit_after>/*
 * Copyright 2009 The Closure Compiler Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.javascript.jscomp;

import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.javascript.jscomp.CompilerOptions.TweakProcessing;
import com.google.javascript.jscomp.deps.ClosureBundler;
import com.google.javascript.jscomp.deps.SourceCodeEscapers;
import com.google.javascript.rhino.Node;
import com.google.javascript.rhino.TokenStream;
import com.google.protobuf.CodedOutputStream;

import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.Flushable;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.StringWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;

import javax.annotation.Nullable;

/**
 * Implementations of AbstractCommandLineRunner translate flags into Java
 * API calls on the Compiler. AbstractCompiler contains common flags and logic
 * to make that happen.
 *
 * This class may be extended and used to create other Java classes
 * that behave the same as running the Compiler from the command line. Example:
 *
 * <pre>
 * class MyCommandLineRunner extends
 *     AbstractCommandLineRunner<MyCompiler, MyOptions> {
 *   MyCommandLineRunner(String[] args) {
 *     super(args);
 *   }
 *
 *   &#064;Override
 *   protected MyOptions createOptions() {
 *     MyOptions options = new MyOptions();
 *     CompilerFlagTranslator.setOptionsFromFlags(options);
 *     addMyCrazyCompilerPassThatOutputsAnExtraFile(options);
 *     return options;
 *   }
 *
 *   &#064;Override
 *   protected MyCompiler createCompiler() {
 *     return new MyCompiler();
 *   }
 *
 *   public static void main(String[] args) {
 *     (new MyCommandLineRunner(args)).run();
 *   }
 * }
 * </pre>
 *
 * @author <EMAIL> (<NAME>)
 */
abstract class AbstractCommandLineRunner<A extends Compiler,
    B extends CompilerOptions> {
  static final DiagnosticType OUTPUT_SAME_AS_INPUT_ERROR = DiagnosticType.error(
      "JSC_OUTPUT_SAME_AS_INPUT_ERROR",
      "Bad output file (already listed as input file): {0}");
  static final DiagnosticType NO_TREE_GENERATED_ERROR = DiagnosticType.error(
      "JSC_NO_TREE_GENERATED_ERROR",
      "Code contains errors. No tree was generated.");

  private final CommandLineConfig config;

  private final PrintStream defaultJsOutput;
  private final PrintStream err;
  private A compiler;

  private Charset inputCharset;

  // NOTE(nicksantos): JSCompiler has always used ASCII as the default
  // output charset. This was done to solve legacy problems with
  // bad proxies, etc. We are not sure if these issues are still problems,
  // and changing the encoding would require a big obnoxious migration plan.
  //
  // New outputs should use outputCharset2, which is how I would have
  // designed this if I had a time machine.
  private Charset outputCharset2;
  private String legacyOutputCharset;

  private boolean testMode = false;
  private Supplier<List<SourceFile>> externsSupplierForTesting = null;
  private Supplier<List<SourceFile>> inputsSupplierForTesting = null;
  private Supplier<List<JSModule>> modulesSupplierForTesting = null;
  private Function<Integer, Boolean> exitCodeReceiverForTesting = null;
  private Map<String, String> rootRelativePathsMap = null;

  private Map<String, String> parsedModuleWrappers = null;

  static final String OUTPUT_MARKER = "%output%";
  private static final String OUTPUT_MARKER_JS_STRING = "%output|jsstring%";

  AbstractCommandLineRunner() {
    this(System.out, System.err);
  }

  AbstractCommandLineRunner(PrintStream out, PrintStream err) {
    this.config = new CommandLineConfig();
    this.defaultJsOutput = Preconditions.checkNotNull(out);
    this.err = Preconditions.checkNotNull(err);
  }

  /**
   * Put the command line runner into test mode. In test mode,
   * all outputs will be blackholed.
   * @param externsSupplier A provider for externs.
   * @param inputsSupplier A provider for source inputs.
   * @param modulesSupplier A provider for modules. Only one of inputsSupplier
   *     and modulesSupplier may be non-null.
   * @param exitCodeReceiver A receiver for the status code that would
   *     have been passed to System.exit in non-test mode.
   */
  @VisibleForTesting
  void enableTestMode(
      Supplier<List<SourceFile>> externsSupplier,
      Supplier<List<SourceFile>> inputsSupplier,
      Supplier<List<JSModule>> modulesSupplier,
      Function<Integer, Boolean> exitCodeReceiver) {
    Preconditions.checkArgument(
        inputsSupplier == null ^ modulesSupplier == null);
    testMode = true;
    this.externsSupplierForTesting = externsSupplier;
    this.inputsSupplierForTesting = inputsSupplier;
    this.modulesSupplierForTesting = modulesSupplier;
    this.exitCodeReceiverForTesting = exitCodeReceiver;
  }

  /**
   * Returns whether we're in test mode.
   */
  protected boolean isInTestMode() {
    return testMode;
  }

  /**
   * Get the command line config, so that it can be initialized.
   */
  protected CommandLineConfig getCommandLineConfig() {
    return config;
  }

  /**
   * Returns the instance of the Compiler to use when {@link #run()} is
   * called.
   */
  protected abstract A createCompiler();

  /**
   * Returns the instance of the Options to use when {@link #run()} is called.
   * createCompiler() is called before createOptions(), so getCompiler()
   * will not return null when createOptions() is called.
   */
  protected abstract B createOptions();

  /**
   * The warning classes that are available from the command-line.
   */
  protected DiagnosticGroups getDiagnosticGroups() {
    if (compiler == null) {
      return new DiagnosticGroups();
    }
    return compiler.getDiagnosticGroups();
  }

  /**
   * A helper function for creating the dependency options object.
   */
  static DependencyOptions createDependencyOptions(
      boolean manageClosureDependencies,
      boolean onlyClosureDependencies,
      boolean processCommonJSModules,
      boolean rewriteEs6Modules,
      List<String> closureEntryPoints)
      throws FlagUsageException {
    if (onlyClosureDependencies) {
      if (closureEntryPoints.isEmpty()) {
        throw new FlagUsageException("When only_closure_dependencies is "
          + "on, you must specify at least one closure_entry_point");
      }

      return new DependencyOptions()
          .setDependencyPruning(true)
          .setDependencySorting(true)
          .setMoocherDropping(true)
          .setEntryPoints(closureEntryPoints);
    } else if (processCommonJSModules) {
      return new DependencyOptions()
        .setDependencyPruning(false)
        .setDependencySorting(true)
        .setMoocherDropping(false)
        .setEntryPoints(closureEntryPoints);
    } else if (rewriteEs6Modules) {
      return new DependencyOptions()
        .setDependencyPruning(false)
        .setDependencySorting(true)
        .setMoocherDropping(false)
        .setEntryPoints(closureEntryPoints);
    } else if (manageClosureDependencies || !closureEntryPoints.isEmpty()) {
      return new DependencyOptions()
          .setDependencyPruning(true)
          .setDependencySorting(true)
          .setMoocherDropping(false)
          .setEntryPoints(closureEntryPoints);
    }
    return null;
  }

  protected abstract void addWhitelistWarningsGuard(
      CompilerOptions options, File whitelistFile);

  /**
   * Sets options based on the configurations set flags API.
   * Called during the run() run() method.
   * If you want to ignore the flags API, or interpret flags your own way,
   * then you should override this method.
   */
  protected void setRunOptions(CompilerOptions options)
      throws FlagUsageException, IOException {
    DiagnosticGroups diagnosticGroups = getDiagnosticGroups();

    if (config.warningGuards != null) {
      for (WarningGuardSpec.Entry entry : config.warningGuards.entries) {
        diagnosticGroups.setWarningLevel(options, entry.groupName, entry.level);
      }
    }

    if (!config.warningsWhitelistFile.isEmpty()) {
      addWhitelistWarningsGuard(options, new File(config.warningsWhitelistFile));
    }

    createDefineOrTweakReplacements(config.define, options, false);

    options.setTweakProcessing(config.tweakProcessing);
    createDefineOrTweakReplacements(config.tweak, options, true);

    DependencyOptions depOptions = createDependencyOptions(
        config.manageClosureDependencies,
        config.onlyClosureDependencies,
        config.processCommonJSModules,
        config.rewriteEs6Modules,
        config.closureEntryPoints);
    if (depOptions != null) {
      options.setDependencyOptions(depOptions);
    }

    options.devMode = config.jscompDevMode;
    options.setCodingConvention(config.codingConvention);
    options.setSummaryDetailLevel(config.summaryDetailLevel);
    options.setTrustedStrings(true);

    legacyOutputCharset = options.outputCharset = getLegacyOutputCharset();
    outputCharset2 = getOutputCharset2();
    inputCharset = getInputCharset();

    if (config.jsOutputFile.length() > 0) {
      if (config.skipNormalOutputs) {
        throw new FlagUsageException("skip_normal_outputs and js_output_file"
            + " cannot be used together.");
      }
    }

    if (config.skipNormalOutputs && config.printAst) {
      throw new FlagUsageException("skip_normal_outputs and print_ast cannot"
          + " be used together.");
    }

    if (config.skipNormalOutputs && config.printTree) {
      throw new FlagUsageException("skip_normal_outputs and print_tree cannot"
          + " be used together.");
    }

    if (config.createSourceMap.length() > 0) {
      options.sourceMapOutputPath = config.createSourceMap;
    }
    options.sourceMapDetailLevel = config.sourceMapDetailLevel;
    options.sourceMapFormat = config.sourceMapFormat;
    options.sourceMapLocationMappings = config.sourceMapLocationMappings;

    ImmutableMap.Builder<String, SourceMapInput> inputSourceMaps
        = new ImmutableMap.Builder<>();
    for (Map.Entry<String, String> files :
             config.sourceMapInputFiles.entrySet()) {
      SourceFile sourceMap = SourceFile.fromFile(files.getValue());
      inputSourceMaps.put(
          files.getKey(), new SourceMapInput(sourceMap));
    }
    options.inputSourceMaps = inputSourceMaps.build();

    if (!config.variableMapInputFile.isEmpty()) {
      options.inputVariableMap =
          VariableMap.load(config.variableMapInputFile);
    }

    if (!config.propertyMapInputFile.isEmpty()) {
      options.inputPropertyMap =
          VariableMap.load(config.propertyMapInputFile);
    }

    if (config.languageIn.length() > 0) {
      CompilerOptions.LanguageMode languageMode =
          CompilerOptions.LanguageMode.fromString(config.languageIn);
      if (languageMode != null) {
        options.setLanguageIn(languageMode);
      } else {
        throw new FlagUsageException("Unknown language `" + config.languageIn
                                     + "' specified.");
      }
    }

    if (config.languageOut.isEmpty()) {
      options.setLanguageOut(options.getLanguageIn());
    } else {
      CompilerOptions.LanguageMode languageMode =
          CompilerOptions.LanguageMode.fromString(config.languageOut);
      if (languageMode != null) {
        options.setLanguageOut(languageMode);
      } else {
        throw new FlagUsageException("Unknown language `" + config.languageOut +
                                     "' specified.");
      }
    }

    if (!config.outputManifests.isEmpty()) {
      Set<String> uniqueNames = Sets.newHashSet();
      for (String filename : config.outputManifests) {
        if (!uniqueNames.add(filename)) {
          throw new FlagUsageException("output_manifest flags specify " +
              "duplicate file names: " + filename);
        }
      }
    }

    if (!config.outputBundles.isEmpty()) {
      Set<String> uniqueNames = Sets.newHashSet();
      for (String filename : config.outputBundles) {
        if (!uniqueNames.add(filename)) {
          throw new FlagUsageException("output_bundle flags specify " +
              "duplicate file names: " + filename);
        }
      }
    }

    options.acceptConstKeyword = config.acceptConstKeyword;
    options.transformAMDToCJSModules = config.transformAMDToCJSModules;
    options.processCommonJSModules = config.processCommonJSModules;
    options.rewriteEs6Modules = config.rewriteEs6Modules;
    options.transpileOnly = config.transpileOnly;
    options.commonJSModulePathPrefix = config.commonJSModulePathPrefix;
    options.angularPass = config.angularPass;
    options.tracer = config.tracerMode;
    options.useNewTypeInference = config.useNewTypeInference;
  }

  protected final A getCompiler() {
    return compiler;
  }

  /**
   * Runs the Compiler and calls System.exit() with the exit status of the
   * compiler.
   */
  public final void run() {
    int result = 0;
    int runs = 1;
    try {
      for (int i = 0; i < runs && result == 0; i++) {
        result = doRun();
      }
    } catch (AbstractCommandLineRunner.FlagUsageException e) {
      System.err.println(e.getMessage());
      result = -1;
    } catch (Throwable t) {
      t.printStackTrace();
      result = -2;
    }

    if (testMode) {
      exitCodeReceiverForTesting.apply(result);
    } else {
      System.exit(result);
    }
  }

  /**
   * Returns the PrintStream for writing errors associated with this
   * AbstractCommandLineRunner.
   */
  protected PrintStream getErrorPrintStream() {
    return err;
  }

  /**
   * An exception thrown when command-line flags are used incorrectly.
   */
  public static class FlagUsageException extends Exception {
    private static final long serialVersionUID = 1L;

    public FlagUsageException(String message) {
      super(message);
    }
  }

  /**
   * Creates inputs from a list of files.
   *
   * Can be overridden by subclasses who want to pull files from different
   * places.
   *
   * @param files A list of filenames
   * @param allowStdIn Whether '-' is allowed appear as a filename to represent
   *        stdin. If true, '-' is only allowed to appear once.
   * @return An array of inputs
   */
  protected List<SourceFile> createInputs(List<String> files,
      boolean allowStdIn) throws FlagUsageException, IOException {
    List<SourceFile> inputs = new ArrayList<>(files.size());
    boolean usingStdin = false;
    for (String filename : files) {
      if (!"-".equals(filename)) {
        SourceFile newFile = SourceFile.fromFile(filename, inputCharset);
        inputs.add(newFile);
      } else {
        if (!allowStdIn) {
          throw new FlagUsageException("Can't specify stdin.");
        }
        if (usingStdin) {
          throw new FlagUsageException("Can't specify stdin twice.");
        }

        if (!config.outputManifests.isEmpty()) {
          throw new FlagUsageException("Manifest files cannot be generated " +
              "when the input is from stdin.");
        }
        if (!config.outputBundles.isEmpty()) {
          throw new FlagUsageException("Bundle files cannot be generated " +
              "when the input is from stdin.");
        }
        inputs.add(SourceFile.fromInputStream("stdin", System.in, inputCharset));
        usingStdin = true;
      }
    }
    return inputs;
  }

  /**
   * Creates JS source code inputs from a list of files.
   */
  private List<SourceFile> createSourceInputs(List<String> files)
      throws FlagUsageException, IOException {
    if (isInTestMode()) {
      return inputsSupplierForTesting.get();
    }
    if (files.isEmpty()) {
      files = Collections.singletonList("-");
    }
    try {
      return createInputs(files, true);
    } catch (FlagUsageException e) {
      throw new FlagUsageException("Bad --js flag. " + e.getMessage());
    }
  }

  /**
   * Creates JS extern inputs from a list of files.
   */
  private List<SourceFile> createExternInputs(List<String> files)
      throws FlagUsageException, IOException {
    if (files.isEmpty()) {
      return ImmutableList.of(SourceFile.fromCode("/dev/null", ""));
    }
    try {
      return createInputs(files, false);
    } catch (FlagUsageException e) {
      throw new FlagUsageException("Bad --externs flag. " + e.getMessage());
    }
  }

  /**
   * Creates module objects from a list of module specifications.
   *
   * @param specs A list of module specifications, not null or empty. The spec
   *        format is: <code>name:num-js-files[:[dep,...][:]]</code>. Module
   *        names must not contain the ':' character.
   * @param jsFiles A list of JS file paths, not null
   * @return An array of module objects
   */
  List<JSModule> createJsModules(
      List<String> specs, List<String> jsFiles)
      throws FlagUsageException, IOException {
    if (isInTestMode()) {
      return modulesSupplierForTesting.get();
    }

    Preconditions.checkState(specs != null);
    Preconditions.checkState(!specs.isEmpty());
    Preconditions.checkState(jsFiles != null);

    List<String> moduleNames = new ArrayList<>(specs.size());
    Map<String, JSModule> modulesByName = Maps.newLinkedHashMap();
    Map<String, Integer> modulesFileCountMap = Maps.newLinkedHashMap();
    int numJsFilesExpected = 0, minJsFilesRequired = 0;
    boolean isFirstModule = true;
    for (String spec : specs) {
      // Format is "<name>:<num-js-files>[:[<dep>,...][:]]".
      String[] parts = spec.split(":");
      if (parts.length < 2 || parts.length > 4) {
        throw new FlagUsageException("Expected 2-4 colon-delimited parts in "
            + "module spec: " + spec);
      }

      // Parse module name.
      String name = parts[0];
      checkModuleName(name);
      if (modulesByName.containsKey(name)) {
        throw new FlagUsageException("Duplicate module name: " + name);
      }
      JSModule module = new JSModule(name);

      if (parts.length > 2) {
        // Parse module dependencies.
        String depList = parts[2];
        if (depList.length() > 0) {
          String[] deps = depList.split(",");
          for (String dep : deps) {
            JSModule other = modulesByName.get(dep);
            if (other == null) {
              throw new FlagUsageException("Module '" + name
                  + "' depends on unknown module '" + dep
                  + "'. Be sure to list modules in dependency order.");
            }
            module.addDependency(other);
          }
        }
      }

      // Parse module inputs.
      int numJsFiles = -1;
      try {
        numJsFiles = Integer.parseInt(parts[1]);
      } catch (NumberFormatException ignored) {
        numJsFiles = -1;
      }

      // We will allow modules of zero input.
      if (numJsFiles < 0) {
        // A size of 'auto' is only allowed on the base module if
        // and it must also be the first module
        if (parts.length == 2 && "auto".equals(parts[1])) {
          if (isFirstModule) {
            numJsFilesExpected = -1;
          } else {
            throw new FlagUsageException("Invalid JS file count '" + parts[1]
                + "' for module: " + name + ". Only the first module may specify " +
                "a size of 'auto' and it must have no dependencies.");
          }
        } else {
          throw new FlagUsageException("Invalid JS file count '" + parts[1]
              + "' for module: " + name);
        }
      } else {
        minJsFilesRequired += numJsFiles;
      }


      if (numJsFilesExpected >= 0) {
        numJsFilesExpected += numJsFiles;
      }

      // Add modules in reverse order so that source files are allocated to
      // modules in reverse order. This allows the first module
      // (presumably the base module) to have a size of 'auto'
      moduleNames.add(0, name);
      modulesFileCountMap.put(name, numJsFiles);
      modulesByName.put(name, module);
    }

    final int totalNumJsFiles = jsFiles.size();

    if (numJsFilesExpected >= 0 || minJsFilesRequired > totalNumJsFiles) {
      if (minJsFilesRequired > totalNumJsFiles) {
        numJsFilesExpected = minJsFilesRequired;
      }

      if (numJsFilesExpected > totalNumJsFiles) {
        throw new FlagUsageException("Not enough JS files specified. Expected "
            + numJsFilesExpected + " but found " + totalNumJsFiles);
      } else if (numJsFilesExpected < totalNumJsFiles) {
        throw new FlagUsageException("Too many JS files specified. Expected "
                + numJsFilesExpected + " but found " + totalNumJsFiles);
      }
    }

    int numJsFilesLeft = totalNumJsFiles, moduleIndex = 0;
    for (String moduleName : moduleNames) {
      // Parse module inputs.
      int numJsFiles = modulesFileCountMap.get(moduleName);
      JSModule module = modulesByName.get(moduleName);

      // Check if the first module specified 'auto' for the number of files
      if (moduleIndex == moduleNames.size() - 1 && numJsFiles == -1) {
        numJsFiles = numJsFilesLeft;
      }

      List<String> moduleJsFiles =
          jsFiles.subList(numJsFilesLeft - numJsFiles, numJsFilesLeft);
      for (SourceFile input : createInputs(moduleJsFiles, false)) {
        module.add(input);
      }
      numJsFilesLeft -= numJsFiles;
      moduleIndex++;
    }

    return Lists.newArrayList(modulesByName.values());
  }

  /**
   * Validates the module name. Can be overridden by subclasses.
   * @param name The module name
{'fraction_non_alphanumeric': 0.09733275032794053, 'fraction_numerical': 0.0027984258854394403, 'mean_word_length': 2.9990382934079385, 'pattern_counts': {'":': 2, '<': 109, '<?xml version=': 0, '>': 123, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6540364', 'n_tokens_mistral': 12767, 'n_tokens_neox': 12497, 'n_words': 3389}