<commit_before>//===--- FrontendTool.cpp - Swift Compiler Frontend -----------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
///
/// \file
/// \brief This is the entry point to the swift -frontend functionality, which
/// implements the core compiler functionality along with a number of additional
/// tools for demonstration and testing purposes.
///
/// This is separate from the rest of libFrontend to reduce the dependencies
/// required by that library.
///
//===----------------------------------------------------------------------===//

#include "swift/FrontendTool/FrontendTool.h"
#include "ImportedModules.h"
#include "ReferenceDependencies.h"
#include "TBD.h"

#include "swift/Strings.h"
#include "swift/Subsystems.h"
#include "swift/AST/ASTScope.h"
#include "swift/AST/DiagnosticsFrontend.h"
#include "swift/AST/DiagnosticsSema.h"
#include "swift/AST/GenericSignatureBuilder.h"
#include "swift/AST/IRGenOptions.h"
#include "swift/AST/ASTMangler.h"
#include "swift/AST/ReferencedNameTracker.h"
#include "swift/AST/TypeRefinementContext.h"
#include "swift/Basic/Dwarf.h"
#include "swift/Basic/Edit.h"
#include "swift/Basic/FileSystem.h"
#include "swift/Basic/JSONSerialization.h"
#include "swift/Basic/LLVMContext.h"
#include "swift/Basic/SourceManager.h"
#include "swift/Basic/Statistic.h"
#include "swift/Basic/Timer.h"
#include "swift/Basic/UUID.h"
#include "swift/Frontend/DiagnosticVerifier.h"
#include "swift/Frontend/Frontend.h"
#include "swift/Frontend/PrintingDiagnosticConsumer.h"
#include "swift/Frontend/SerializedDiagnosticConsumer.h"
#include "swift/Immediate/Immediate.h"
#include "swift/Index/IndexRecord.h"
#include "swift/Option/Options.h"
#include "swift/Migrator/FixitFilter.h"
#include "swift/Migrator/Migrator.h"
#include "swift/PrintAsObjC/PrintAsObjC.h"
#include "swift/Serialization/SerializationOptions.h"
#include "swift/Serialization/SerializedModuleLoader.h"
#include "swift/SILOptimizer/PassManager/Passes.h"
#include "swift/Syntax/Serialization/SyntaxSerialization.h"
#include "swift/Syntax/SyntaxNodes.h"

// FIXME: We're just using CompilerInstance::createOutputFile.
// This API should be sunk down to LLVM.
#include "clang/Frontend/CompilerInstance.h"
#include "clang/AST/ASTContext.h"
#include "clang/APINotes/Types.h"

#include "llvm/ADT/Statistic.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/Option/Option.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/YAMLTraits.h"
#include "llvm/Target/TargetMachine.h"

#include <deque>
#include <memory>
#include <unordered_set>

#if !defined(_MSC_VER) && !defined(__MINGW32__)
#include <unistd.h>
#else
#include <io.h>
#endif

using namespace swift;

static std::string displayName(StringRef MainExecutablePath) {
  std::string Name = llvm::sys::path::stem(MainExecutablePath);
  Name += " -frontend";
  return Name;
}

/// Emits a Make-style dependencies file.
static bool emitMakeDependencies(DiagnosticEngine &diags,
                                 DependencyTracker &depTracker,
                                 const FrontendOptions &opts,
                                 const InputFile &input) {
  std::error_code EC;
  llvm::raw_fd_ostream out(opts.DependenciesFilePath, EC,
                           llvm::sys::fs::F_None);

  if (out.has_error() || EC) {
    diags.diagnose(SourceLoc(), diag::error_opening_output,
                   opts.DependenciesFilePath, EC.message());
    out.clear_error();
    return true;
  }

  // Declare a helper for escaping file names for use in Makefiles.
  llvm::SmallString<256> pathBuf;
  auto escape = [&](StringRef raw) -> StringRef {
    pathBuf.clear();

    static const char badChars[] = " $#:\n";
    size_t prev = 0;
    for (auto index = raw.find_first_of(badChars); index != StringRef::npos;
         index = raw.find_first_of(badChars, index+1)) {
      pathBuf.append(raw.slice(prev, index));
      if (raw[index] == '$')
        pathBuf.push_back('$');
      else
        pathBuf.push_back('\\');
      prev = index;
    }
    pathBuf.append(raw.substr(prev));
    return pathBuf;
  };

  // FIXME: Xcode can't currently handle multiple targets in a single
  // dependency line.
  opts.forAllOutputPaths(input, [&](StringRef targetName) {
    out << escape(targetName) << " :";
    // First include all other files in the module. Make-style dependencies
    // need to be conservative!
    for (auto const &path :
         reversePathSortedFilenames(opts.InputsAndOutputs.getInputFilenames()))
      out << ' ' << escape(path);
    // Then print dependencies we've picked up during compilation.
    for (auto const &path :
           reversePathSortedFilenames(depTracker.getDependencies()))
      out << ' ' << escape(path);
    out << '\n';
  });

  return false;
}

static bool emitMakeDependencies(DiagnosticEngine &diags,
                                 DependencyTracker &depTracker,
                                 const FrontendOptions &opts) {
  bool hadError = false;
  opts.InputsAndOutputs.forEachInputProducingSupplementaryOutput(
      [&](const InputFile &f) -> void {
        hadError = emitMakeDependencies(diags, depTracker, opts, f) || hadError;
      });
  return hadError;
}

namespace {
struct LoadedModuleTraceFormat {
  std::string Name;
  std::string Arch;
  std::vector<std::string> SwiftModules;
};
}

namespace swift {
namespace json {
template <> struct ObjectTraits<LoadedModuleTraceFormat> {
  static void mapping(Output &out, LoadedModuleTraceFormat &contents) {
    out.mapRequired("name", contents.Name);
    out.mapRequired("arch", contents.Arch);
    out.mapRequired("swiftmodules", contents.SwiftModules);
  }
};
}
}

static bool emitLoadedModuleTraceIfNeeded(ASTContext &ctxt,
                                          DependencyTracker &depTracker,
                                          const FrontendOptions &opts) {
  if (opts.LoadedModuleTracePath.empty())
    return false;
  std::error_code EC;
  llvm::raw_fd_ostream out(opts.LoadedModuleTracePath, EC,
                           llvm::sys::fs::F_Append);

  if (out.has_error() || EC) {
    ctxt.Diags.diagnose(SourceLoc(), diag::error_opening_output,
                        opts.LoadedModuleTracePath, EC.message());
    out.clear_error();
    return true;
  }

  llvm::SmallVector<std::string, 16> swiftModules;

  // Canonicalise all the paths by opening them.
  for (auto &dep : depTracker.getDependencies()) {
    llvm::SmallString<256> buffer;
    StringRef realPath;
    int FD;
    // FIXME: appropriate error handling
    if (llvm::sys::fs::openFileForRead(dep, FD, &buffer)) {
      // Couldn't open the file now, so let's just assume the old path was
      // canonical (enough).
      realPath = dep;
    } else {
      realPath = buffer.str();
      // Not much we can do about failing to close.
      (void)close(FD);
    }

    // Decide if this is a swiftmodule based on the extension of the raw
    // dependency path, as the true file may have a different one.
    auto ext = llvm::sys::path::extension(dep);
    if (ext.startswith(".") &&
        ext.drop_front() == SERIALIZED_MODULE_EXTENSION) {
      swiftModules.push_back(realPath);
    }
  }

  LoadedModuleTraceFormat trace = {
      /*name=*/opts.ModuleName,
      /*arch=*/ctxt.LangOpts.Target.getArchName(),
      /*swiftmodules=*/reversePathSortedFilenames(swiftModules)};

  // raw_fd_ostream is unbuffered, and we may have multiple processes writing,
  // so first write the whole thing into memory and dump out that buffer to the
  // file.
  std::string stringBuffer;
  {
    llvm::raw_string_ostream memoryBuffer(stringBuffer);
    json::Output jsonOutput(memoryBuffer, /*PrettyPrint=*/false);
    json::jsonize(jsonOutput, trace, /*Required=*/true);
  }
  stringBuffer += "\n";

  out << stringBuffer;

  return true;
}


/// Gets an output stream for the provided output filename, or diagnoses to the
/// provided AST Context and returns null if there was an error getting the
/// stream.
static std::unique_ptr<llvm::raw_fd_ostream>
getFileOutputStream(StringRef OutputFilename, ASTContext &Ctx) {
  std::error_code errorCode;
  auto os = llvm::make_unique<llvm::raw_fd_ostream>(
              OutputFilename, errorCode, llvm::sys::fs::F_None);
  if (errorCode) {
    Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_output,
                       OutputFilename, errorCode.message());
    return nullptr;
  }
  return os;
}

/// Writes the Syntax tree to the given file
static bool emitSyntax(SourceFile *SF, LangOptions &LangOpts,
                       SourceManager &SM, StringRef OutputFilename) {
  auto bufferID = SF->getBufferID();
  assert(bufferID && "frontend should have a buffer ID "
         "for the main source file");

  auto os = getFileOutputStream(OutputFilename, SF->getASTContext());
  if (!os) return true;

  json::Output jsonOut(*os);
  auto Root = SF->getSyntaxRoot().getRaw();
  jsonOut << *Root;
  *os << "\n";
  return false;
}

/// Writes SIL out to the given file.
static bool writeSIL(SILModule &SM, ModuleDecl *M, bool EmitVerboseSIL,
                     StringRef OutputFilename, bool SortSIL) {
  auto OS = getFileOutputStream(OutputFilename, M->getASTContext());
  if (!OS) return true;
  SM.print(*OS, EmitVerboseSIL, M, SortSIL);
  return false;
}

static bool writeSIL(SILModule &SM, CompilerInstance &Instance,
                     CompilerInvocation &Invocation) {
  const FrontendOptions &opts = Invocation.getFrontendOptions();
  return writeSIL(SM, Instance.getMainModule(), opts.EmitVerboseSIL,
                  opts.InputsAndOutputs.getSingleOutputFilename(),
                  opts.EmitSortedSIL);
}

static bool printAsObjC(const std::string &outputPath, ModuleDecl *M,
                        StringRef bridgingHeader, bool moduleIsPublic) {
  using namespace llvm::sys;

  clang::CompilerInstance Clang;

  std::string tmpFilePath;
  std::error_code EC;
  std::unique_ptr<llvm::raw_pwrite_stream> out =
    Clang.createOutputFile(outputPath, EC,
                           /*Binary=*/false,
                           /*RemoveFileOnSignal=*/true,
                           /*BaseInput=*/"",
                           path::extension(outputPath),
                           /*UseTemporary=*/true,
                           /*CreateMissingDirectories=*/false,
                           /*ResultPathName=*/nullptr,
                           &tmpFilePath);

  if (!out) {
    M->getASTContext().Diags.diagnose(SourceLoc(), diag::error_opening_output,
                                      tmpFilePath, EC.message());
    return true;
  }

  auto requiredAccess = moduleIsPublic ? AccessLevel::Public
                                       : AccessLevel::Internal;
  bool hadError = printAsObjC(*out, M, bridgingHeader, requiredAccess);
  out->flush();

  EC = swift::moveFileIfDifferent(tmpFilePath, outputPath);
  if (EC) {
    M->getASTContext().Diags.diagnose(SourceLoc(), diag::error_opening_output,
                                      outputPath, EC.message());
    return true;
  }

  return hadError;
}

/// Returns the OutputKind for the given Action.
static IRGenOutputKind getOutputKind(FrontendOptions::ActionType Action) {
  switch (Action) {
  case FrontendOptions::ActionType::EmitIR:
    return IRGenOutputKind::LLVMAssembly;
  case FrontendOptions::ActionType::EmitBC:
    return IRGenOutputKind::LLVMBitcode;
  case FrontendOptions::ActionType::EmitAssembly:
    return IRGenOutputKind::NativeAssembly;
  case FrontendOptions::ActionType::EmitObject:
    return IRGenOutputKind::ObjectFile;
  case FrontendOptions::ActionType::Immediate:
    return IRGenOutputKind::Module;
  default:
    llvm_unreachable("Unknown ActionType which requires IRGen");
    return IRGenOutputKind::ObjectFile;
  }
}

namespace {

/// If there is an error with fixits it writes the fixits as edits in json
/// format.
class JSONFixitWriter
  : public DiagnosticConsumer, public migrator::FixitFilter {
  std::string FixitsOutputPath;
  std::unique_ptr<llvm::raw_ostream> OSPtr;
  bool FixitAll;
  std::vector<SingleEdit> AllEdits;

public:
  JSONFixitWriter(std::string fixitsOutputPath,
                  const DiagnosticOptions &DiagOpts)
    : FixitsOutputPath(fixitsOutputPath),
      FixitAll(DiagOpts.FixitCodeForAllDiagnostics) {}

private:
  void handleDiagnostic(SourceManager &SM, SourceLoc Loc,
                        DiagnosticKind Kind,
                        StringRef FormatString,
                        ArrayRef<DiagnosticArgument> FormatArgs,
                        const DiagnosticInfo &Info) override {
    if (!(FixitAll || shouldTakeFixit(Kind, Info)))
      return;
    for (const auto &Fix : Info.FixIts) {
      AllEdits.push_back({SM, Fix.getRange(), Fix.getText()});
    }
  }

  bool finishProcessing() override {
    std::error_code EC;
    std::unique_ptr<llvm::raw_fd_ostream> OS;
    OS.reset(new llvm::raw_fd_ostream(FixitsOutputPath,
                                      EC,
                                      llvm::sys::fs::F_None));
    if (EC) {
      // Create a temporary diagnostics engine to print the error to stderr.
      SourceManager dummyMgr;
      DiagnosticEngine DE(dummyMgr);
      PrintingDiagnosticConsumer PDC;
      DE.addConsumer(PDC);
      DE.diagnose(SourceLoc(), diag::cannot_open_file,
                  FixitsOutputPath, EC.message());
      return true;
    }

    swift::writeEditsInJson(llvm::makeArrayRef(AllEdits), *OS);
    return false;
  }
};

} // anonymous namespace

// This is a separate function so that it shows up in stack traces.
LLVM_ATTRIBUTE_NOINLINE
static void debugFailWithAssertion() {
  // Per the user's request, this assertion should always fail in
  // builds with assertions enabled.

  // This should not be converted to llvm_unreachable, as those are
  // treated as optimization hints in builds where they turn into
  // __builtin_unreachable().
  assert((0) && "This is an assertion!");
}

// This is a separate function so that it shows up in stack traces.
LLVM_ATTRIBUTE_NOINLINE
static void debugFailWithCrash() {
  LLVM_BUILTIN_TRAP;
}

static bool emitIndexData(SourceFile *PrimarySourceFile,
      const CompilerInvocation &Invocation,
      CompilerInstance &Instance);

static void countStatsOfSourceFile(UnifiedStatsReporter &Stats,
                                   CompilerInstance &Instance,
                                   SourceFile *SF) {
  auto &C = Stats.getFrontendCounters();
  auto &SM = Instance.getSourceMgr();
  C.NumDecls += SF->Decls.size();
  C.NumLocalTypeDecls += SF->LocalTypeDecls.size();
  C.NumObjCMethods += SF->ObjCMethods.size();
  C.NumInfixOperators += SF->InfixOperators.size();
  C.NumPostfixOperators += SF->PostfixOperators.size();
  C.NumPrefixOperators += SF->PrefixOperators.size();
  C.NumPrecedenceGroups += SF->PrecedenceGroups.size();
  C.NumUsedConformances += SF->getUsedConformances().size();

  auto bufID = SF->getBufferID();
  if (bufID.hasValue()) {
    C.NumSourceLines +=
      SM.getEntireTextForBuffer(bufID.getValue()).count('\n');
  }
}

static void countStatsPostSema(UnifiedStatsReporter &Stats,
                               CompilerInstance& Instance) {
  auto &C = Stats.getFrontendCounters();
  auto &SM = Instance.getSourceMgr();
  C.NumSourceBuffers = SM.getLLVMSourceMgr().getNumBuffers();
  C.NumLinkLibraries = Instance.getLinkLibraries().size();

  auto const &AST = Instance.getASTContext();
  C.NumLoadedModules = AST.LoadedModules.size();
  C.NumImportedExternalDefinitions = AST.ExternalDefinitions.size();
  C.NumASTBytesAllocated = AST.getAllocator().getBytesAllocated();

  if (auto *D = Instance.getDependencyTracker()) {
    C.NumDependencies = D->getDependencies().size();
  }

  if (auto *R = Instance.getReferencedNameTracker()) {
    C.NumReferencedTopLevelNames = R->getTopLevelNames().size();
    C.NumReferencedDynamicNames = R->getDynamicLookupNames().size();
    C.NumReferencedMemberNames = R->getUsedMembers().size();
  }

  if (!Instance.getPrimarySourceFiles().empty()) {
    for (auto SF : Instance.getPrimarySourceFiles())
      countStatsOfSourceFile(Stats, Instance, SF);
  } else if (auto *M = Instance.getMainModule()) {
    // No primary source file, but a main module; this is WMO-mode
    for (auto *F : M->getFiles()) {
      if (auto *SF = dyn_cast<SourceFile>(F)) {
        countStatsOfSourceFile(Stats, Instance, SF);
      }
    }
  }
}

static void countStatsPostSILGen(UnifiedStatsReporter &Stats,
                                 const SILModule& Module) {
  auto &C = Stats.getFrontendCounters();
  // FIXME: calculate these in constant time, via the dense maps.
  C.NumSILGenFunctions = Module.getFunctionList().size();
  C.NumSILGenVtables = Module.getVTableList().size();
  C.NumSILGenWitnessTables = Module.getWitnessTableList().size();
  C.NumSILGenDefaultWitnessTables = Module.getDefaultWitnessTableList().size();
  C.NumSILGenGlobalVariables = Module.getSILGlobalList().size();
}

static void countStatsPostSILOpt(UnifiedStatsReporter &Stats,
                                 const SILModule& Module) {
  auto &C = Stats.getFrontendCounters();
  // FIXME: calculate these in constant time, via the dense maps.
  C.NumSILOptFunctions = Module.getFunctionList().size();
  C.NumSILOptVtables = Module.getVTableList().size();
  C.NumSILOptWitnessTables = Module.getWitnessTableList().size();
  C.NumSILOptDefaultWitnessTables = Module.getDefaultWitnessTableList().size();
  C.NumSILOptGlobalVariables = Module.getSILGlobalList().size();
}

static std::unique_ptr<llvm::raw_fd_ostream>
createOptRecordFile(StringRef Filename, DiagnosticEngine &DE) {
  if (Filename.empty())
    return nullptr;

  std::error_code EC;
  auto File = llvm::make_unique<llvm::raw_fd_ostream>(Filename, EC,
                                                      llvm::sys::fs::F_None);
  if (EC) {
    DE.diagnose(SourceLoc(), diag::cannot_open_file, Filename, EC.message());
    return nullptr;
  }
  return File;
}

struct PostSILGenInputs {
  std::unique_ptr<SILModule> TheSILModule;
  bool ASTGuaranteedToCorrespondToSIL;
  ModuleOrSourceFile ModuleOrPrimarySourceFile;
};

static bool precompileBridgingHeader(CompilerInvocation &Invocation,
                                     CompilerInstance &Instance) {
  auto clangImporter = static_cast<ClangImporter *>(
      Instance.getASTContext().getClangModuleLoader());
  auto &ImporterOpts = Invocation.getClangImporterOptions();
  auto &PCHOutDir = ImporterOpts.PrecompiledHeaderOutputDir;
  if (!PCHOutDir.empty()) {
    ImporterOpts.BridgingHeader =
        Invocation.getFrontendOptions()
            .InputsAndOutputs.getFilenameOfFirstInput();
    // Create or validate a persistent PCH.
    auto SwiftPCHHash = Invocation.getPCHHash();
    auto PCH = clangImporter->getOrCreatePCH(ImporterOpts, SwiftPCHHash);
    return !PCH.hasValue();
  }
  return clangImporter->emitBridgingPCH(
      Invocation.getFrontendOptions()
          .InputsAndOutputs.getFilenameOfFirstInput(),
      Invocation.getFrontendOptions()
          .InputsAndOutputs.getSingleOutputFilename());
}

static bool compileLLVMIR(CompilerInvocation &Invocation,
                          CompilerInstance &Instance,
                          UnifiedStatsReporter *Stats) {
  auto &LLVMContext = getGlobalLLVMContext();

  // Load in bitcode file.
  assert(Invocation.getFrontendOptions().InputsAndOutputs.hasSingleInput() &&
         "We expect a single input for bitcode input!");
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileBufOrErr =
      llvm::MemoryBuffer::getFileOrSTDIN(
          Invocation.getFrontendOptions()
              .InputsAndOutputs.getFilenameOfFirstInput());
  if (!FileBufOrErr) {
    Instance.getASTContext().Diags.diagnose(
        SourceLoc(), diag::error_open_input_file,
        Invocation.getFrontendOptions()
            .InputsAndOutputs.getFilenameOfFirstInput(),
        FileBufOrErr.getError().message());
    return true;
  }
  llvm::MemoryBuffer *MainFile = FileBufOrErr.get().get();

  llvm::SMDiagnostic Err;
  std::unique_ptr<llvm::Module> Module =
      llvm::parseIR(MainFile->getMemBufferRef(), Err, LLVMContext);
  if (!Module) {
    // TODO: Translate from the diagnostic info to the SourceManager location
    // if available.
    Instance.getASTContext().Diags.diagnose(
        SourceLoc(), diag::error_parse_input_file,
        Invocation.getFrontendOptions()
            .InputsAndOutputs.getFilenameOfFirstInput(),
        Err.getMessage());
    return true;
  }
  IRGenOptions &IRGenOpts = Invocation.getIRGenOptions();
  // TODO: remove once the frontend understands what action it should perform
  IRGenOpts.OutputKind =
      getOutputKind(Invocation.getFrontendOptions().RequestedAction);

  return performLLVM(IRGenOpts, Instance.getASTContext(), Module.get(), Stats);
}

static void verifyGenericSignaturesIfNeeded(CompilerInvocation &Invocation,
                                            ASTContext &Context) {
  auto verifyGenericSignaturesInModule =
      Invocation.getFrontendOptions().VerifyGenericSignaturesInModule;
  if (verifyGenericSignaturesInModule.empty())
    return;
  if (auto module = Context.getModuleByName(verifyGenericSignaturesInModule))
    GenericSignatureBuilder::verifyGenericSignaturesInModule(module);
}

static void dumpOneScopeMapLocation(unsigned bufferID,
                                    std::pair<unsigned, unsigned> lineColumn,
                                    SourceManager &sourceMgr, ASTScope &scope) {
  SourceLoc loc =
      sourceMgr.getLocForLineCol(bufferID, lineColumn.first, lineColumn.second);
  if (loc.isInvalid())
    return;

  llvm::errs() << "***Scope at " << lineColumn.first << ":" << lineColumn.second
               << "***\n";
  auto locScope = scope.findInnermostEnclosingScope(loc);
  locScope->print(llvm::errs(), 0, false, false);

  // Dump the AST context, too.
  if (auto dc = locScope->getDeclContext()) {
    dc->printContext(llvm::errs());
  }

  // Grab the local bindings introduced by this scope.
  auto localBindings = locScope->getLocalBindings();
  if (!localBindings.empty()) {
    llvm::errs() << "Local bindings: ";
    interleave(localBindings.begin(), localBindings.end(),
               [&](ValueDecl *value) { llvm::errs() << value->getFullName(); },
               [&]() { llvm::errs() << " "; });
    llvm::errs() << "\n";
  }
}

static void dumpAndPrintScopeMap(CompilerInvocation &Invocation,
                                 CompilerInstance &Instance, SourceFile *SF) {
  ASTScope &scope = SF->getScope();

  if (Invocation.getFrontendOptions().DumpScopeMapLocations.empty()) {
    scope.expandAll();
  } else if (auto bufferID = SF->getBufferID()) {
    SourceManager &sourceMgr = Instance.getSourceMgr();
    // Probe each of the locations, and dump what we find.
    for (auto lineColumn :
         Invocation.getFrontendOptions().DumpScopeMapLocations)
      dumpOneScopeMapLocation(*bufferID, lineColumn, sourceMgr, scope);

    llvm::errs() << "***Complete scope map***\n";
  }
  // Print the resulting map.
  scope.print(llvm::errs());
}

static SourceFile *
getSourceFileOfCodeToBeGenerated(CompilerInvocation &Invocation,
                                 CompilerInstance &Instance) {
  SourceFile *SF = Instance.getPrimarySourceFile();
  if (!SF) {
    SourceFileKind Kind = Invocation.getSourceFileKind();
    SF = &Instance.getMainModule()->getMainSourceFile(Kind);
  }
  return SF;
}

/// We may have been told to dump the AST (either after parsing or
/// type-checking, which is already differentiated in
/// CompilerInstance::performSema()), so dump or print the main source file and
/// return.

static Optional<bool> dumpASTIfNeeded(CompilerInvocation &Invocation,
                                      CompilerInstance &Instance) {
  FrontendOptions &opts = Invocation.getFrontendOptions();
  FrontendOptions::ActionType Action = opts.RequestedAction;
  ASTContext &Context = Instance.getASTContext();
  switch (Action) {
  default:
    return None;

  case FrontendOptions::ActionType::PrintAST:
    getSourceFileOfCodeToBeGenerated(Invocation, Instance)
        ->print(llvm::outs(), PrintOptions::printEverything());
    break;

  case FrontendOptions::ActionType::DumpScopeMaps:
    dumpAndPrintScopeMap(
        Invocation, Instance,
        getSourceFileOfCodeToBeGenerated(Invocation, Instance));
    break;

  case FrontendOptions::ActionType::DumpTypeRefinementContexts:
    getSourceFileOfCodeToBeGenerated(Invocation, Instance)
        ->getTypeRefinementContext()
        ->dump(llvm::errs(), Context.SourceMgr);
    break;

  case FrontendOptions::ActionType::DumpInterfaceHash:
    getSourceFileOfCodeToBeGenerated(Invocation, Instance)
        ->dumpInterfaceHash(llvm::errs());
    break;

  case FrontendOptions::ActionType::EmitSyntax:
    emitSyntax(getSourceFileOfCodeToBeGenerated(Invocation, Instance),
               Invocation.getLangOptions(), Instance.getSourceMgr(),
               opts.InputsAndOutputs.getSingleOutputFilename());
    break;

  case FrontendOptions::ActionType::DumpParse:
  case FrontendOptions::ActionType::DumpAST:
    getSourceFileOfCodeToBeGenerated(Invocation, Instance)->dump();
    break;

  case FrontendOptions::ActionType::EmitImportedModules:
    emitImportedModules(Context, Instance.getMainModule(), opts);
    break;
  }
  return Context.hadError();
}

static void emitReferenceDependenciesIfNeeded(CompilerInvocation &Invocation,
                                              CompilerInstance &Instance) {
  if (Invocation.getFrontendOptions().ReferenceDependenciesFilePath.empty())
    return;
  if (Instance.getPrimarySourceFiles().empty()) {
    Instance.getASTContext().Diags.diagnose(
        SourceLoc(), diag::emit_reference_dependencies_without_primary_file);
    return;
  }
  for (auto *SF : Instance.getPrimarySourceFiles()) {
    emitReferenceDependencies(Instance.getASTContext().Diags, SF,
                              *Instance.getDependencyTracker(),
                              Invocation.getFrontendOptions());
  }
}

static bool writeTBDIfNeeded(CompilerInvocation &Invocation,
                             CompilerInstance &Instance) {
  StringRef TBDPath = Invocation.getFrontendOptions().TBDPath;
  if (TBDPath.empty())
    return false;
  auto installName = Invocation.getFrontendOptions().TBDInstallName.empty()
                         ? "lib" + Invocation.getModuleName().str() + ".dylib"
                         : Invocation.getFrontendOptions().TBDInstallName;

  return writeTBD(Instance.getMainModule(),
                  Invocation.getSILOptions().hasMultipleIGMs(), TBDPath,
                  installName);
}

static std::deque<PostSILGenInputs>
generateSILModules(CompilerInvocation &Invocation, CompilerInstance &Instance) {
  auto mod = Instance.getMainModule();
  if (auto SM = Instance.takeSILModule()) {
    std::deque<PostSILGenInputs> PSGIs;
    PSGIs.push_back(PostSILGenInputs{std::move(SM), false, mod});
    return PSGIs;
  }

  SILOptions &SILOpts = Invocation.getSILOptions();
  FrontendOptions &opts = Invocation.getFrontendOptions();
  auto fileIsSIB = [](const FileUnit *File) -> bool {
    auto SASTF = dyn_cast<SerializedASTFile>(File);
    return SASTF && SASTF->isSIB();
  };

  if (!opts.InputsAndOutputs.hasPrimaryInputs()) {
    // If there are no primary inputs the compiler is in WMO mode and builds one
    // SILModule for the entire module.
    auto SM = performSILGeneration(mod, SILOpts, true);
    std::deque<PostSILGenInputs> PSGIs;
    PSGIs.push_back(PostSILGenInputs{
        std::move(SM), llvm::none_of(mod->getFiles(), fileIsSIB), mod});
    return PSGIs;
  }
  // If there are primary source files, build a separate SILModule for
  // each source file, and run the remaining SILOpt-Serialize-IRGen-LLVM
  // once for each such input.
  if (auto *PrimaryFile = Instance.getPrimarySourceFile()) {
    auto SM = performSILGeneration(*PrimaryFile, SILOpts, None);
    std::deque<PostSILGenInputs> PSGIs;
    PSGIs.push_back(PostSILGenInputs{std::move(SM), true, PrimaryFile});
    return PSGIs;
  }
  // If there are primary inputs but no primary _source files_, there might be
  // a primary serialized input.
  std::deque<PostSILGenInputs> PSGIs;
  for (FileUnit *fileUnit : mod->getFiles()) {
    if (auto SASTF = dyn_cast<SerializedASTFile>(fileUnit))
      if (Invocation.getFrontendOptions().InputsAndOutputs.isInputPrimary(
              SASTF->getFilename())) {
        assert(PSGIs.empty() && "Can only handle one primary AST input");
        auto SM = performSILGeneration(*SASTF, SILOpts, None);
        PSGIs.push_back(
            PostSILGenInputs{std::move(SM), !fileIsSIB(SASTF), mod});
      }
  }
  return PSGIs;
}

static bool performCompileStepsPostSILGen(CompilerInstance &Instance,
                                          CompilerInvocation &Invocation,
                                          std::unique_ptr<SILModule> SM,
                                          bool astGuaranteedToCorrespondToSIL,
                                          ModuleOrSourceFile MSF,
                                          bool moduleIsPublic,
                                          int &ReturnValue,
                                          FrontendObserver *observer,
                                          UnifiedStatsReporter *Stats);

/// Performs the compile requested by the user.
/// \param Instance Will be reset after performIRGeneration when the verifier
///                 mode is NoVerify and there were no errors.
/// \returns true on error
static bool performCompile(CompilerInstance &Instance,
                           CompilerInvocation &Invocation,
                           ArrayRef<const char *> Args,
                           int &ReturnValue,
                           FrontendObserver *observer,
                           UnifiedStatsReporter *Stats) {
  FrontendOptions opts = Invocation.getFrontendOptions();
  FrontendOptions::ActionType Action = opts.RequestedAction;

  if (Action == FrontendOptions::ActionType::EmitSyntax) {
    Instance.getASTContext().LangOpts.KeepSyntaxInfoInSourceFile = true;
    Instance.getASTContext().LangOpts.VerifySyntaxTree = true;
  }

  // We've been asked to precompile a bridging header; we want to
  // avoid touching any other inputs and just parse, emit and exit.
  if (Action == FrontendOptions::ActionType::EmitPCH)
    return precompileBridgingHeader(Invocation, Instance);

    if (Invocation.getInputKind() == InputFileKind::IFK_LLVM_IR)
      return compileLLVMIR(Invocation, Instance, Stats);

  ReferencedNameTracker nameTracker;
  if (!opts.ReferenceDependenciesFilePath.empty())
    Instance.setReferencedNameTracker(&nameTracker);

  if (FrontendOptions::shouldActionOnlyParse(Action))
    Instance.performParseOnly();
  else
    Instance.performSema();

  if (Action == FrontendOptions::ActionType::Parse)
    return Instance.getASTContext().hadError();

  if (observer) {
    observer->performedSemanticAnalysis(Instance);
  }

  if (Stats) {
    countStatsPostSema(*Stats, Instance);
  }

  FrontendOptions::DebugCrashMode CrashMode = opts.CrashMode;
  if (CrashMode == FrontendOptions::DebugCrashMode::AssertAfterParse)
    debugFailWithAssertion();
  else if (CrashMode == FrontendOptions::DebugCrashMode::CrashAfterParse)
    debugFailWithCrash();

  ASTContext &Context = Instance.getASTContext();

  verifyGenericSignaturesIfNeeded(Invocation, Context);

  if (Invocation.getMigratorOptions().shouldRunMigrator()) {
    migrator::updateCodeAndEmitRemap(&Instance, Invocation);
  }

  if (Action == FrontendOptions::ActionType::REPL) {
    runREPL(Instance, ProcessCmdLine(Args.begin(), Args.end()),
            Invocation.getParseStdlib());
    return Context.hadError();
  }

  if (auto r = dumpASTIfNeeded(Invocation, Instance))
    return *r;

  // If we were asked to print Clang stats, do so.
  if (opts.PrintClangStats && Context.getClangModuleLoader())
    Context.getClangModuleLoader()->printStatistics();

  if (!opts.DependenciesFilePath.empty())
    (void)emitMakeDependencies(Context.Diags, *Instance.getDependencyTracker(),
                               opts);

  emitReferenceDependenciesIfNeeded(Invocation, Instance);

  (void)emitLoadedModuleTraceIfNeeded(Context, *Instance.getDependencyTracker(),
                                      opts);

  bool shouldIndex = !opts.IndexStorePath.empty();

  if (Context.hadError()) {
    if (shouldIndex) {
      //  Emit the index store data even if there were compiler errors.
      if (emitIndexData(Instance.getPrimarySourceFile(),
                        Invocation, Instance))
        return true;
    }
    return true;
  }

  // FIXME: This is still a lousy approximation of whether the module file will
  // be externally consumed.
  bool moduleIsPublic =
      !Instance.getMainModule()->hasEntryPoint() &&
      opts.ImplicitObjCHeaderPath.empty() &&
      !Context.LangOpts.EnableAppExtensionRestrictions;

  // We've just been told to perform a typecheck, so we can return now.
  if (Action == FrontendOptions::ActionType::Typecheck) {
    if (!opts.ObjCHeaderOutputPath.empty())
      return printAsObjC(opts.ObjCHeaderOutputPath, Instance.getMainModule(),
                         opts.ImplicitObjCHeaderPath, moduleIsPublic);
    if (shouldIndex) {
      if (emitIndexData(Instance.getPrimarySourceFile(),
                        Invocation, Instance))
        return true;
    }
    return Context.hadError();
  }

  if (writeTBDIfNeeded(Invocation, Instance))
    return true;

  assert(Action >= FrontendOptions::ActionType::EmitSILGen &&
         "All actions not requiring SILGen must have been handled!");

  std::deque<PostSILGenInputs> PSGIs = generateSILModules(Invocation, Instance);

  while (!PSGIs.empty()) {
    auto PSGI = std::move(PSGIs.front());
    PSGIs.pop_front();
    if (performCompileStepsPostSILGen(Instance, Invocation,
                                      std::move(PSGI.TheSILModule),
                                      PSGI.ASTGuaranteedToCorrespondToSIL,
                                      PSGI.ModuleOrPrimarySourceFile,
                                      moduleIsPublic,
                                      ReturnValue, observer, Stats))
      return true;
  }
  return false;
}

/// If we are asked to link all, link all.
static void linkAllIfNeeded(const CompilerInvocation &Invocation,
                            SILModule *SM) {
  if (Invocation.getSILOptions().LinkMode == SILOptions::LinkAll)
    performSILLinking(SM, true);
}

/// Perform "stable" optimizations that are invariant across compiler versions.
static bool performMandatorySILPasses(CompilerInvocation &Invocation,
                                      SILModule *SM,
                                      FrontendObserver *observer) {
  if (Invocation.getFrontendOptions().RequestedAction ==
      FrontendOptions::ActionType::MergeModules) {
    // Don't run diagnostic passes at all.
  } else if (!Invocation.getDiagnosticOptions().SkipDiagnosticPasses) {
    if (runSILDiagnosticPasses(*SM))
      return true;

    if (observer) {
      observer->performedSILDiagnostics(*SM);
    }
  } else {
    // Even if we are not supposed to run the diagnostic passes, we still need
    // to run the ownership evaluator.
    if (runSILOwnershipEliminatorPass(*SM))
      return true;
  }

  linkAllIfNeeded(Invocation, SM);

  if (Invocation.getSILOptions().MergePartialModules)
    SM->linkAllFromCurrentModule();
  return false;
}

static SerializationOptions
computeSerializationOptions(const CompilerInvocation &Invocation,
                            bool moduleIsPublic) {
  const FrontendOptions &opts = Invocation.getFrontendOptions();

  SerializationOptions serializationOpts;
  serializationOpts.OutputPath = opts.ModuleOutputPath.c_str();
  serializationOpts.DocOutputPath = opts.ModuleDocOutputPath.c_str();
  serializationOpts.GroupInfoPath = opts.GroupInfoPath.c_str();
  if (opts.SerializeBridgingHeader)
    serializationOpts.ImportedHeader = opts.ImplicitObjCHeaderPath;
  serializationOpts.ModuleLinkName = opts.ModuleLinkName;
  serializationOpts.ExtraClangOptions =
      Invocation.getClangImporterOptions().ExtraArgs;
  serializationOpts.EnableNestedTypeLookupTable =
      opts.EnableSerializationNestedTypeLookupTable;
  if (!Invocation.getIRGenOptions().ForceLoadSymbolName.empty())
    serializationOpts.AutolinkForceLoad = true;

  // Options contain information about the developer's computer,
  // so only serialize them if the module isn't going to be shipped to
  // the public.
  serializationOpts.SerializeOptionsForDebugging =
      !moduleIsPublic || opts.AlwaysSerializeDebuggingOptions;

  return serializationOpts;
}

/// Perform SIL optimization passes if optimizations haven't been disabled.
/// These may change across compiler versions.
static void performSILOptimizations(CompilerInvocation &Invocation,
                                    SILModule *SM) {
  SharedTimer timer("SIL optimization");
  if (Invocation.getFrontendOptions().RequestedAction ==
          FrontendOptions::ActionType::MergeModules ||
      !Invocation.getSILOptions().shouldOptimize()) {
    runSILPassesForOnone(*SM);
    return;
  }
  runSILOptPreparePasses(*SM);

  StringRef CustomPipelinePath =
      Invocation.getSILOptions().ExternalPassPipelineFilename;
  if (!CustomPipelinePath.empty()) {
    runSILOptimizationPassesWithFileSpecification(*SM, CustomPipelinePath);
  } else {
    runSILOptimizationPasses(*SM);
  }
}

/// Get the main source file's private discriminator and attach it to
/// the compile unit's flags.
static void setPrivateDiscriminatorIfNeeded(IRGenOptions &IRGenOpts,
                                            ModuleOrSourceFile MSF) {
  if (IRGenOpts.DebugInfoKind == IRGenDebugInfoKind::None ||
      !MSF.is<SourceFile *>())
    return;
  Identifier PD = MSF.get<SourceFile *>()->getPrivateDiscriminator();
  if (!PD.empty())
    IRGenOpts.DWARFDebugFlags += (" -private-discriminator " + PD.str()).str();
}

static bool serializeSIB(FrontendOptions &opts, SILModule *SM,
                         ASTContext &Context, ModuleOrSourceFile MSF) {
  const std::string &moduleOutputPath = opts.ModuleOutputPath;
  assert(!moduleOutputPath.empty() && "must have an output path");

  SerializationOptions serializationOpts;
  serializationOpts.OutputPath = moduleOutputPath.c_str();
  serializationOpts.SerializeAllSIL = true;
  serializationOpts.IsSIB = true;
  serialize(MSF, serializationOpts, SM);
  return Context.hadError();
}

static bool performCompileStepsPostSILGen(CompilerInstance &Instance,
                                          CompilerInvocation &Invocation,
                                          std::unique_ptr<SILModule> SM,
                                          bool astGuaranteedToCorrespondToSIL,
                                          ModuleOrSourceFile MSF,
                                          bool moduleIsPublic,
                                          int &ReturnValue,
                                          FrontendObserver *observer,
                                          UnifiedStatsReporter *Stats) {

  FrontendOptions opts = Invocation.getFrontendOptions();
  FrontendOptions::ActionType Action = opts.RequestedAction;
  ASTContext &Context = Instance.getASTContext();
  SILOptions &SILOpts = Invocation.getSILOptions();
  IRGenOptions &IRGenOpts = Invocation.getIRGenOptions();
  bool shouldIndex = !opts.IndexStorePath.empty();

  if (observer) {
    observer->performedSILGeneration(*SM);
  }
  if (Stats) {
    countStatsPostSILGen(*Stats, *SM);
  }

  // We've been told to emit SIL after SILGen, so write it now.
  if (Action == FrontendOptions::ActionType::EmitSILGen) {
    linkAllIfNeeded(Invocation, SM.get());
    return writeSIL(*SM, Instance, Invocation);
  }

  if (Action == FrontendOptions::ActionType::EmitSIBGen) {
    linkAllIfNeeded(Invocation, SM.get());
    serializeSIB(Invocation.getFrontendOptions(), SM.get(),
                 Instance.getASTContext(), MSF);
    return Context.hadError();
  }

  std::unique_ptr<llvm::raw_fd_ostream> OptRecordFile =
      createOptRecordFile(SILOpts.OptRecordFile, Instance.getDiags());
  if (OptRecordFile)
    SM->setOptRecordStream(llvm::make_unique<llvm::yaml::Output>(
                               *OptRecordFile, &Instance.getSourceMgr()),
                           std::move(OptRecordFile));

  if (performMandatorySILPasses(Invocation, SM.get(), observer))
    return true;

  {
    SharedTimer timer("SIL verification, pre-optimization");
    SM->verify();
  }

  // This is the action to be used to serialize SILModule.
  // It may be invoked multiple times, but it will perform
  // serialization only once. The serialization may either happen
  // after high-level optimizations or after all optimizations are
  // done, depending on the compiler setting.

  auto SerializeSILModuleAction = [&]() {
    if (opts.ModuleOutputPath.empty())
      return;

    SerializationOptions serializationOpts =
        computeSerializationOptions(Invocation, moduleIsPublic);
    serialize(MSF, serializationOpts, SM.get());
  };

  // Set the serialization action, so that the SIL module
  // can be serialized at any moment, e.g. during the optimization pipeline.
  SM->setSerializeSILAction(SerializeSILModuleAction);

  performSILOptimizations(Invocation, SM.get());

  if (observer) {
    observer->performedSILOptimization(*SM);
  }
  if (Stats) {
    countStatsPostSILOpt(*Stats, *SM);
  }

  {
    SharedTimer timer("SIL verification, post-optimization");
    SM->verify();
  }

  // Gather instruction counts if we are asked to do so.
  if (SM->getOptions().PrintInstCounts) {
    performSILInstCount(&*SM);
  }

  setPrivateDiscriminatorIfNeeded(IRGenOpts, MSF);

  if (!opts.ObjCHeaderOutputPath.empty()) {
    (void)printAsObjC(opts.ObjCHeaderOutputPath, Instance.getMainModule(),
                      opts.ImplicitObjCHeaderPath, moduleIsPublic);
  }

  if (Action == FrontendOptions::ActionType::EmitSIB)
    return serializeSIB(Invocation.getFrontendOptions(), SM.get(),
                        Instance.getASTContext(), MSF);

  if (!opts.ModuleOutputPath.empty() || !opts.ModuleDocOutputPath.empty()) {
    // Serialize the SILModule if it was not serialized yet.
    if (!SM.get()->isSerialized())
      SM.get()->serialize();
    if (Action == FrontendOptions::ActionType::MergeModules ||
        Action == FrontendOptions::ActionType::EmitModuleOnly) {
      if (shouldIndex) {
        if (emitIndexData(MSF.dyn_cast<SourceFile*>(),
                          Invocation, Instance))
          return true;
      }
      return Context.hadError();
    }
  }

  assert(Action >= FrontendOptions::ActionType::EmitSIL &&
         "All actions not requiring SILPasses must have been handled!");

  // We've been told to write canonical SIL, so write it now.
  if (Action == FrontendOptions::ActionType::EmitSIL)
    return writeSIL(*SM, Instance, Invocation);

  assert(Action >= FrontendOptions::ActionType::Immediate &&
         "All actions not requiring IRGen must have been handled!");
  assert(Action != FrontendOptions::ActionType::REPL &&
         "REPL mode must be handled immediately after Instance->performSema()");<commit_msg>Extract ensureSILModuleIsSerialized.<commit_after>//===--- FrontendTool.cpp - Swift Compiler Frontend -----------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
///
/// \file
/// \brief This is the entry point to the swift -frontend functionality, which
/// implements the core compiler functionality along with a number of additional
/// tools for demonstration and testing purposes.
///
/// This is separate from the rest of libFrontend to reduce the dependencies
/// required by that library.
///
//===----------------------------------------------------------------------===//

#include "swift/FrontendTool/FrontendTool.h"
#include "ImportedModules.h"
#include "ReferenceDependencies.h"
#include "TBD.h"

#include "swift/Strings.h"
#include "swift/Subsystems.h"
#include "swift/AST/ASTScope.h"
#include "swift/AST/DiagnosticsFrontend.h"
#include "swift/AST/DiagnosticsSema.h"
#include "swift/AST/GenericSignatureBuilder.h"
#include "swift/AST/IRGenOptions.h"
#include "swift/AST/ASTMangler.h"
#include "swift/AST/ReferencedNameTracker.h"
#include "swift/AST/TypeRefinementContext.h"
#include "swift/Basic/Dwarf.h"
#include "swift/Basic/Edit.h"
#include "swift/Basic/FileSystem.h"
#include "swift/Basic/JSONSerialization.h"
#include "swift/Basic/LLVMContext.h"
#include "swift/Basic/SourceManager.h"
#include "swift/Basic/Statistic.h"
#include "swift/Basic/Timer.h"
#include "swift/Basic/UUID.h"
#include "swift/Frontend/DiagnosticVerifier.h"
#include "swift/Frontend/Frontend.h"
#include "swift/Frontend/PrintingDiagnosticConsumer.h"
#include "swift/Frontend/SerializedDiagnosticConsumer.h"
#include "swift/Immediate/Immediate.h"
#include "swift/Index/IndexRecord.h"
#include "swift/Option/Options.h"
#include "swift/Migrator/FixitFilter.h"
#include "swift/Migrator/Migrator.h"
#include "swift/PrintAsObjC/PrintAsObjC.h"
#include "swift/Serialization/SerializationOptions.h"
#include "swift/Serialization/SerializedModuleLoader.h"
#include "swift/SILOptimizer/PassManager/Passes.h"
#include "swift/Syntax/Serialization/SyntaxSerialization.h"
#include "swift/Syntax/SyntaxNodes.h"

// FIXME: We're just using CompilerInstance::createOutputFile.
// This API should be sunk down to LLVM.
#include "clang/Frontend/CompilerInstance.h"
#include "clang/AST/ASTContext.h"
#include "clang/APINotes/Types.h"

#include "llvm/ADT/Statistic.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/Option/Option.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/YAMLTraits.h"
#include "llvm/Target/TargetMachine.h"

#include <deque>
#include <memory>
#include <unordered_set>

#if !defined(_MSC_VER) && !defined(__MINGW32__)
#include <unistd.h>
#else
#include <io.h>
#endif

using namespace swift;

static std::string displayName(StringRef MainExecutablePath) {
  std::string Name = llvm::sys::path::stem(MainExecutablePath);
  Name += " -frontend";
  return Name;
}

/// Emits a Make-style dependencies file.
static bool emitMakeDependencies(DiagnosticEngine &diags,
                                 DependencyTracker &depTracker,
                                 const FrontendOptions &opts,
                                 const InputFile &input) {
  std::error_code EC;
  llvm::raw_fd_ostream out(opts.DependenciesFilePath, EC,
                           llvm::sys::fs::F_None);

  if (out.has_error() || EC) {
    diags.diagnose(SourceLoc(), diag::error_opening_output,
                   opts.DependenciesFilePath, EC.message());
    out.clear_error();
    return true;
  }

  // Declare a helper for escaping file names for use in Makefiles.
  llvm::SmallString<256> pathBuf;
  auto escape = [&](StringRef raw) -> StringRef {
    pathBuf.clear();

    static const char badChars[] = " $#:\n";
    size_t prev = 0;
    for (auto index = raw.find_first_of(badChars); index != StringRef::npos;
         index = raw.find_first_of(badChars, index+1)) {
      pathBuf.append(raw.slice(prev, index));
      if (raw[index] == '$')
        pathBuf.push_back('$');
      else
        pathBuf.push_back('\\');
      prev = index;
    }
    pathBuf.append(raw.substr(prev));
    return pathBuf;
  };

  // FIXME: Xcode can't currently handle multiple targets in a single
  // dependency line.
  opts.forAllOutputPaths(input, [&](StringRef targetName) {
    out << escape(targetName) << " :";
    // First include all other files in the module. Make-style dependencies
    // need to be conservative!
    for (auto const &path :
         reversePathSortedFilenames(opts.InputsAndOutputs.getInputFilenames()))
      out << ' ' << escape(path);
    // Then print dependencies we've picked up during compilation.
    for (auto const &path :
           reversePathSortedFilenames(depTracker.getDependencies()))
      out << ' ' << escape(path);
    out << '\n';
  });

  return false;
}

static bool emitMakeDependencies(DiagnosticEngine &diags,
                                 DependencyTracker &depTracker,
                                 const FrontendOptions &opts) {
  bool hadError = false;
  opts.InputsAndOutputs.forEachInputProducingSupplementaryOutput(
      [&](const InputFile &f) -> void {
        hadError = emitMakeDependencies(diags, depTracker, opts, f) || hadError;
      });
  return hadError;
}

namespace {
struct LoadedModuleTraceFormat {
  std::string Name;
  std::string Arch;
  std::vector<std::string> SwiftModules;
};
}

namespace swift {
namespace json {
template <> struct ObjectTraits<LoadedModuleTraceFormat> {
  static void mapping(Output &out, LoadedModuleTraceFormat &contents) {
    out.mapRequired("name", contents.Name);
    out.mapRequired("arch", contents.Arch);
    out.mapRequired("swiftmodules", contents.SwiftModules);
  }
};
}
}

static bool emitLoadedModuleTraceIfNeeded(ASTContext &ctxt,
                                          DependencyTracker &depTracker,
                                          const FrontendOptions &opts) {
  if (opts.LoadedModuleTracePath.empty())
    return false;
  std::error_code EC;
  llvm::raw_fd_ostream out(opts.LoadedModuleTracePath, EC,
                           llvm::sys::fs::F_Append);

  if (out.has_error() || EC) {
    ctxt.Diags.diagnose(SourceLoc(), diag::error_opening_output,
                        opts.LoadedModuleTracePath, EC.message());
    out.clear_error();
    return true;
  }

  llvm::SmallVector<std::string, 16> swiftModules;

  // Canonicalise all the paths by opening them.
  for (auto &dep : depTracker.getDependencies()) {
    llvm::SmallString<256> buffer;
    StringRef realPath;
    int FD;
    // FIXME: appropriate error handling
    if (llvm::sys::fs::openFileForRead(dep, FD, &buffer)) {
      // Couldn't open the file now, so let's just assume the old path was
      // canonical (enough).
      realPath = dep;
    } else {
      realPath = buffer.str();
      // Not much we can do about failing to close.
      (void)close(FD);
    }

    // Decide if this is a swiftmodule based on the extension of the raw
    // dependency path, as the true file may have a different one.
    auto ext = llvm::sys::path::extension(dep);
    if (ext.startswith(".") &&
        ext.drop_front() == SERIALIZED_MODULE_EXTENSION) {
      swiftModules.push_back(realPath);
    }
  }

  LoadedModuleTraceFormat trace = {
      /*name=*/opts.ModuleName,
      /*arch=*/ctxt.LangOpts.Target.getArchName(),
      /*swiftmodules=*/reversePathSortedFilenames(swiftModules)};

  // raw_fd_ostream is unbuffered, and we may have multiple processes writing,
  // so first write the whole thing into memory and dump out that buffer to the
  // file.
  std::string stringBuffer;
  {
    llvm::raw_string_ostream memoryBuffer(stringBuffer);
    json::Output jsonOutput(memoryBuffer, /*PrettyPrint=*/false);
    json::jsonize(jsonOutput, trace, /*Required=*/true);
  }
  stringBuffer += "\n";

  out << stringBuffer;

  return true;
}


/// Gets an output stream for the provided output filename, or diagnoses to the
/// provided AST Context and returns null if there was an error getting the
/// stream.
static std::unique_ptr<llvm::raw_fd_ostream>
getFileOutputStream(StringRef OutputFilename, ASTContext &Ctx) {
  std::error_code errorCode;
  auto os = llvm::make_unique<llvm::raw_fd_ostream>(
              OutputFilename, errorCode, llvm::sys::fs::F_None);
  if (errorCode) {
    Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_output,
                       OutputFilename, errorCode.message());
    return nullptr;
  }
  return os;
}

/// Writes the Syntax tree to the given file
static bool emitSyntax(SourceFile *SF, LangOptions &LangOpts,
                       SourceManager &SM, StringRef OutputFilename) {
  auto bufferID = SF->getBufferID();
  assert(bufferID && "frontend should have a buffer ID "
         "for the main source file");

  auto os = getFileOutputStream(OutputFilename, SF->getASTContext());
  if (!os) return true;

  json::Output jsonOut(*os);
  auto Root = SF->getSyntaxRoot().getRaw();
  jsonOut << *Root;
  *os << "\n";
  return false;
}

/// Writes SIL out to the given file.
static bool writeSIL(SILModule &SM, ModuleDecl *M, bool EmitVerboseSIL,
                     StringRef OutputFilename, bool SortSIL) {
  auto OS = getFileOutputStream(OutputFilename, M->getASTContext());
  if (!OS) return true;
  SM.print(*OS, EmitVerboseSIL, M, SortSIL);
  return false;
}

static bool writeSIL(SILModule &SM, CompilerInstance &Instance,
                     CompilerInvocation &Invocation) {
  const FrontendOptions &opts = Invocation.getFrontendOptions();
  return writeSIL(SM, Instance.getMainModule(), opts.EmitVerboseSIL,
                  opts.InputsAndOutputs.getSingleOutputFilename(),
                  opts.EmitSortedSIL);
}

static bool printAsObjC(const std::string &outputPath, ModuleDecl *M,
                        StringRef bridgingHeader, bool moduleIsPublic) {
  using namespace llvm::sys;

  clang::CompilerInstance Clang;

  std::string tmpFilePath;
  std::error_code EC;
  std::unique_ptr<llvm::raw_pwrite_stream> out =
    Clang.createOutputFile(outputPath, EC,
                           /*Binary=*/false,
                           /*RemoveFileOnSignal=*/true,
                           /*BaseInput=*/"",
                           path::extension(outputPath),
                           /*UseTemporary=*/true,
                           /*CreateMissingDirectories=*/false,
                           /*ResultPathName=*/nullptr,
                           &tmpFilePath);

  if (!out) {
    M->getASTContext().Diags.diagnose(SourceLoc(), diag::error_opening_output,
                                      tmpFilePath, EC.message());
    return true;
  }

  auto requiredAccess = moduleIsPublic ? AccessLevel::Public
                                       : AccessLevel::Internal;
  bool hadError = printAsObjC(*out, M, bridgingHeader, requiredAccess);
  out->flush();

  EC = swift::moveFileIfDifferent(tmpFilePath, outputPath);
  if (EC) {
    M->getASTContext().Diags.diagnose(SourceLoc(), diag::error_opening_output,
                                      outputPath, EC.message());
    return true;
  }

  return hadError;
}

/// Returns the OutputKind for the given Action.
static IRGenOutputKind getOutputKind(FrontendOptions::ActionType Action) {
  switch (Action) {
  case FrontendOptions::ActionType::EmitIR:
    return IRGenOutputKind::LLVMAssembly;
  case FrontendOptions::ActionType::EmitBC:
    return IRGenOutputKind::LLVMBitcode;
  case FrontendOptions::ActionType::EmitAssembly:
    return IRGenOutputKind::NativeAssembly;
  case FrontendOptions::ActionType::EmitObject:
    return IRGenOutputKind::ObjectFile;
  case FrontendOptions::ActionType::Immediate:
    return IRGenOutputKind::Module;
  default:
    llvm_unreachable("Unknown ActionType which requires IRGen");
    return IRGenOutputKind::ObjectFile;
  }
}

namespace {

/// If there is an error with fixits it writes the fixits as edits in json
/// format.
class JSONFixitWriter
  : public DiagnosticConsumer, public migrator::FixitFilter {
  std::string FixitsOutputPath;
  std::unique_ptr<llvm::raw_ostream> OSPtr;
  bool FixitAll;
  std::vector<SingleEdit> AllEdits;

public:
  JSONFixitWriter(std::string fixitsOutputPath,
                  const DiagnosticOptions &DiagOpts)
    : FixitsOutputPath(fixitsOutputPath),
      FixitAll(DiagOpts.FixitCodeForAllDiagnostics) {}

private:
  void handleDiagnostic(SourceManager &SM, SourceLoc Loc,
                        DiagnosticKind Kind,
                        StringRef FormatString,
                        ArrayRef<DiagnosticArgument> FormatArgs,
                        const DiagnosticInfo &Info) override {
    if (!(FixitAll || shouldTakeFixit(Kind, Info)))
      return;
    for (const auto &Fix : Info.FixIts) {
      AllEdits.push_back({SM, Fix.getRange(), Fix.getText()});
    }
  }

  bool finishProcessing() override {
    std::error_code EC;
    std::unique_ptr<llvm::raw_fd_ostream> OS;
    OS.reset(new llvm::raw_fd_ostream(FixitsOutputPath,
                                      EC,
                                      llvm::sys::fs::F_None));
    if (EC) {
      // Create a temporary diagnostics engine to print the error to stderr.
      SourceManager dummyMgr;
      DiagnosticEngine DE(dummyMgr);
      PrintingDiagnosticConsumer PDC;
      DE.addConsumer(PDC);
      DE.diagnose(SourceLoc(), diag::cannot_open_file,
                  FixitsOutputPath, EC.message());
      return true;
    }

    swift::writeEditsInJson(llvm::makeArrayRef(AllEdits), *OS);
    return false;
  }
};

} // anonymous namespace

// This is a separate function so that it shows up in stack traces.
LLVM_ATTRIBUTE_NOINLINE
static void debugFailWithAssertion() {
  // Per the user's request, this assertion should always fail in
  // builds with assertions enabled.

  // This should not be converted to llvm_unreachable, as those are
  // treated as optimization hints in builds where they turn into
  // __builtin_unreachable().
  assert((0) && "This is an assertion!");
}

// This is a separate function so that it shows up in stack traces.
LLVM_ATTRIBUTE_NOINLINE
static void debugFailWithCrash() {
  LLVM_BUILTIN_TRAP;
}

static bool emitIndexData(SourceFile *PrimarySourceFile,
      const CompilerInvocation &Invocation,
      CompilerInstance &Instance);

static void countStatsOfSourceFile(UnifiedStatsReporter &Stats,
                                   CompilerInstance &Instance,
                                   SourceFile *SF) {
  auto &C = Stats.getFrontendCounters();
  auto &SM = Instance.getSourceMgr();
  C.NumDecls += SF->Decls.size();
  C.NumLocalTypeDecls += SF->LocalTypeDecls.size();
  C.NumObjCMethods += SF->ObjCMethods.size();
  C.NumInfixOperators += SF->InfixOperators.size();
  C.NumPostfixOperators += SF->PostfixOperators.size();
  C.NumPrefixOperators += SF->PrefixOperators.size();
  C.NumPrecedenceGroups += SF->PrecedenceGroups.size();
  C.NumUsedConformances += SF->getUsedConformances().size();

  auto bufID = SF->getBufferID();
  if (bufID.hasValue()) {
    C.NumSourceLines +=
      SM.getEntireTextForBuffer(bufID.getValue()).count('\n');
  }
}

static void countStatsPostSema(UnifiedStatsReporter &Stats,
                               CompilerInstance& Instance) {
  auto &C = Stats.getFrontendCounters();
  auto &SM = Instance.getSourceMgr();
  C.NumSourceBuffers = SM.getLLVMSourceMgr().getNumBuffers();
  C.NumLinkLibraries = Instance.getLinkLibraries().size();

  auto const &AST = Instance.getASTContext();
  C.NumLoadedModules = AST.LoadedModules.size();
  C.NumImportedExternalDefinitions = AST.ExternalDefinitions.size();
  C.NumASTBytesAllocated = AST.getAllocator().getBytesAllocated();

  if (auto *D = Instance.getDependencyTracker()) {
    C.NumDependencies = D->getDependencies().size();
  }

  if (auto *R = Instance.getReferencedNameTracker()) {
    C.NumReferencedTopLevelNames = R->getTopLevelNames().size();
    C.NumReferencedDynamicNames = R->getDynamicLookupNames().size();
    C.NumReferencedMemberNames = R->getUsedMembers().size();
  }

  if (!Instance.getPrimarySourceFiles().empty()) {
    for (auto SF : Instance.getPrimarySourceFiles())
      countStatsOfSourceFile(Stats, Instance, SF);
  } else if (auto *M = Instance.getMainModule()) {
    // No primary source file, but a main module; this is WMO-mode
    for (auto *F : M->getFiles()) {
      if (auto *SF = dyn_cast<SourceFile>(F)) {
        countStatsOfSourceFile(Stats, Instance, SF);
      }
    }
  }
}

static void countStatsPostSILGen(UnifiedStatsReporter &Stats,
                                 const SILModule& Module) {
  auto &C = Stats.getFrontendCounters();
  // FIXME: calculate these in constant time, via the dense maps.
  C.NumSILGenFunctions = Module.getFunctionList().size();
  C.NumSILGenVtables = Module.getVTableList().size();
  C.NumSILGenWitnessTables = Module.getWitnessTableList().size();
  C.NumSILGenDefaultWitnessTables = Module.getDefaultWitnessTableList().size();
  C.NumSILGenGlobalVariables = Module.getSILGlobalList().size();
}

static void countStatsPostSILOpt(UnifiedStatsReporter &Stats,
                                 const SILModule& Module) {
  auto &C = Stats.getFrontendCounters();
  // FIXME: calculate these in constant time, via the dense maps.
  C.NumSILOptFunctions = Module.getFunctionList().size();
  C.NumSILOptVtables = Module.getVTableList().size();
  C.NumSILOptWitnessTables = Module.getWitnessTableList().size();
  C.NumSILOptDefaultWitnessTables = Module.getDefaultWitnessTableList().size();
  C.NumSILOptGlobalVariables = Module.getSILGlobalList().size();
}

static std::unique_ptr<llvm::raw_fd_ostream>
createOptRecordFile(StringRef Filename, DiagnosticEngine &DE) {
  if (Filename.empty())
    return nullptr;

  std::error_code EC;
  auto File = llvm::make_unique<llvm::raw_fd_ostream>(Filename, EC,
                                                      llvm::sys::fs::F_None);
  if (EC) {
    DE.diagnose(SourceLoc(), diag::cannot_open_file, Filename, EC.message());
    return nullptr;
  }
  return File;
}

struct PostSILGenInputs {
  std::unique_ptr<SILModule> TheSILModule;
  bool ASTGuaranteedToCorrespondToSIL;
  ModuleOrSourceFile ModuleOrPrimarySourceFile;
};

static bool precompileBridgingHeader(CompilerInvocation &Invocation,
                                     CompilerInstance &Instance) {
  auto clangImporter = static_cast<ClangImporter *>(
      Instance.getASTContext().getClangModuleLoader());
  auto &ImporterOpts = Invocation.getClangImporterOptions();
  auto &PCHOutDir = ImporterOpts.PrecompiledHeaderOutputDir;
  if (!PCHOutDir.empty()) {
    ImporterOpts.BridgingHeader =
        Invocation.getFrontendOptions()
            .InputsAndOutputs.getFilenameOfFirstInput();
    // Create or validate a persistent PCH.
    auto SwiftPCHHash = Invocation.getPCHHash();
    auto PCH = clangImporter->getOrCreatePCH(ImporterOpts, SwiftPCHHash);
    return !PCH.hasValue();
  }
  return clangImporter->emitBridgingPCH(
      Invocation.getFrontendOptions()
          .InputsAndOutputs.getFilenameOfFirstInput(),
      Invocation.getFrontendOptions()
          .InputsAndOutputs.getSingleOutputFilename());
}

static bool compileLLVMIR(CompilerInvocation &Invocation,
                          CompilerInstance &Instance,
                          UnifiedStatsReporter *Stats) {
  auto &LLVMContext = getGlobalLLVMContext();

  // Load in bitcode file.
  assert(Invocation.getFrontendOptions().InputsAndOutputs.hasSingleInput() &&
         "We expect a single input for bitcode input!");
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileBufOrErr =
      llvm::MemoryBuffer::getFileOrSTDIN(
          Invocation.getFrontendOptions()
              .InputsAndOutputs.getFilenameOfFirstInput());
  if (!FileBufOrErr) {
    Instance.getASTContext().Diags.diagnose(
        SourceLoc(), diag::error_open_input_file,
        Invocation.getFrontendOptions()
            .InputsAndOutputs.getFilenameOfFirstInput(),
        FileBufOrErr.getError().message());
    return true;
  }
  llvm::MemoryBuffer *MainFile = FileBufOrErr.get().get();

  llvm::SMDiagnostic Err;
  std::unique_ptr<llvm::Module> Module =
      llvm::parseIR(MainFile->getMemBufferRef(), Err, LLVMContext);
  if (!Module) {
    // TODO: Translate from the diagnostic info to the SourceManager location
    // if available.
    Instance.getASTContext().Diags.diagnose(
        SourceLoc(), diag::error_parse_input_file,
        Invocation.getFrontendOptions()
            .InputsAndOutputs.getFilenameOfFirstInput(),
        Err.getMessage());
    return true;
  }
  IRGenOptions &IRGenOpts = Invocation.getIRGenOptions();
  // TODO: remove once the frontend understands what action it should perform
  IRGenOpts.OutputKind =
      getOutputKind(Invocation.getFrontendOptions().RequestedAction);

  return performLLVM(IRGenOpts, Instance.getASTContext(), Module.get(), Stats);
}

static void verifyGenericSignaturesIfNeeded(CompilerInvocation &Invocation,
                                            ASTContext &Context) {
  auto verifyGenericSignaturesInModule =
      Invocation.getFrontendOptions().VerifyGenericSignaturesInModule;
  if (verifyGenericSignaturesInModule.empty())
    return;
  if (auto module = Context.getModuleByName(verifyGenericSignaturesInModule))
    GenericSignatureBuilder::verifyGenericSignaturesInModule(module);
}

static void dumpOneScopeMapLocation(unsigned bufferID,
                                    std::pair<unsigned, unsigned> lineColumn,
                                    SourceManager &sourceMgr, ASTScope &scope) {
  SourceLoc loc =
      sourceMgr.getLocForLineCol(bufferID, lineColumn.first, lineColumn.second);
  if (loc.isInvalid())
    return;

  llvm::errs() << "***Scope at " << lineColumn.first << ":" << lineColumn.second
               << "***\n";
  auto locScope = scope.findInnermostEnclosingScope(loc);
  locScope->print(llvm::errs(), 0, false, false);

  // Dump the AST context, too.
  if (auto dc = locScope->getDeclContext()) {
    dc->printContext(llvm::errs());
  }

  // Grab the local bindings introduced by this scope.
  auto localBindings = locScope->getLocalBindings();
  if (!localBindings.empty()) {
    llvm::errs() << "Local bindings: ";
    interleave(localBindings.begin(), localBindings.end(),
               [&](ValueDecl *value) { llvm::errs() << value->getFullName(); },
               [&]() { llvm::errs() << " "; });
    llvm::errs() << "\n";
  }
}

static void dumpAndPrintScopeMap(CompilerInvocation &Invocation,
                                 CompilerInstance &Instance, SourceFile *SF) {
  ASTScope &scope = SF->getScope();

  if (Invocation.getFrontendOptions().DumpScopeMapLocations.empty()) {
    scope.expandAll();
  } else if (auto bufferID = SF->getBufferID()) {
    SourceManager &sourceMgr = Instance.getSourceMgr();
    // Probe each of the locations, and dump what we find.
    for (auto lineColumn :
         Invocation.getFrontendOptions().DumpScopeMapLocations)
      dumpOneScopeMapLocation(*bufferID, lineColumn, sourceMgr, scope);

    llvm::errs() << "***Complete scope map***\n";
  }
  // Print the resulting map.
  scope.print(llvm::errs());
}

static SourceFile *
getSourceFileOfCodeToBeGenerated(CompilerInvocation &Invocation,
                                 CompilerInstance &Instance) {
  SourceFile *SF = Instance.getPrimarySourceFile();
  if (!SF) {
    SourceFileKind Kind = Invocation.getSourceFileKind();
    SF = &Instance.getMainModule()->getMainSourceFile(Kind);
  }
  return SF;
}

/// We may have been told to dump the AST (either after parsing or
/// type-checking, which is already differentiated in
/// CompilerInstance::performSema()), so dump or print the main source file and
/// return.

static Optional<bool> dumpASTIfNeeded(CompilerInvocation &Invocation,
                                      CompilerInstance &Instance) {
  FrontendOptions &opts = Invocation.getFrontendOptions();
  FrontendOptions::ActionType Action = opts.RequestedAction;
  ASTContext &Context = Instance.getASTContext();
  switch (Action) {
  default:
    return None;

  case FrontendOptions::ActionType::PrintAST:
    getSourceFileOfCodeToBeGenerated(Invocation, Instance)
        ->print(llvm::outs(), PrintOptions::printEverything());
    break;

  case FrontendOptions::ActionType::DumpScopeMaps:
    dumpAndPrintScopeMap(
        Invocation, Instance,
        getSourceFileOfCodeToBeGenerated(Invocation, Instance));
    break;

  case FrontendOptions::ActionType::DumpTypeRefinementContexts:
    getSourceFileOfCodeToBeGenerated(Invocation, Instance)
        ->getTypeRefinementContext()
        ->dump(llvm::errs(), Context.SourceMgr);
    break;

  case FrontendOptions::ActionType::DumpInterfaceHash:
    getSourceFileOfCodeToBeGenerated(Invocation, Instance)
        ->dumpInterfaceHash(llvm::errs());
    break;

  case FrontendOptions::ActionType::EmitSyntax:
    emitSyntax(getSourceFileOfCodeToBeGenerated(Invocation, Instance),
               Invocation.getLangOptions(), Instance.getSourceMgr(),
               opts.InputsAndOutputs.getSingleOutputFilename());
    break;

  case FrontendOptions::ActionType::DumpParse:
  case FrontendOptions::ActionType::DumpAST:
    getSourceFileOfCodeToBeGenerated(Invocation, Instance)->dump();
    break;

  case FrontendOptions::ActionType::EmitImportedModules:
    emitImportedModules(Context, Instance.getMainModule(), opts);
    break;
  }
  return Context.hadError();
}

static void emitReferenceDependenciesIfNeeded(CompilerInvocation &Invocation,
                                              CompilerInstance &Instance) {
  if (Invocation.getFrontendOptions().ReferenceDependenciesFilePath.empty())
    return;
  if (Instance.getPrimarySourceFiles().empty()) {
    Instance.getASTContext().Diags.diagnose(
        SourceLoc(), diag::emit_reference_dependencies_without_primary_file);
    return;
  }
  for (auto *SF : Instance.getPrimarySourceFiles()) {
    emitReferenceDependencies(Instance.getASTContext().Diags, SF,
                              *Instance.getDependencyTracker(),
                              Invocation.getFrontendOptions());
  }
}

static bool writeTBDIfNeeded(CompilerInvocation &Invocation,
                             CompilerInstance &Instance) {
  StringRef TBDPath = Invocation.getFrontendOptions().TBDPath;
  if (TBDPath.empty())
    return false;
  auto installName = Invocation.getFrontendOptions().TBDInstallName.empty()
                         ? "lib" + Invocation.getModuleName().str() + ".dylib"
                         : Invocation.getFrontendOptions().TBDInstallName;

  return writeTBD(Instance.getMainModule(),
                  Invocation.getSILOptions().hasMultipleIGMs(), TBDPath,
                  installName);
}

static std::deque<PostSILGenInputs>
generateSILModules(CompilerInvocation &Invocation, CompilerInstance &Instance) {
  auto mod = Instance.getMainModule();
  if (auto SM = Instance.takeSILModule()) {
    std::deque<PostSILGenInputs> PSGIs;
    PSGIs.push_back(PostSILGenInputs{std::move(SM), false, mod});
    return PSGIs;
  }

  SILOptions &SILOpts = Invocation.getSILOptions();
  FrontendOptions &opts = Invocation.getFrontendOptions();
  auto fileIsSIB = [](const FileUnit *File) -> bool {
    auto SASTF = dyn_cast<SerializedASTFile>(File);
    return SASTF && SASTF->isSIB();
  };

  if (!opts.InputsAndOutputs.hasPrimaryInputs()) {
    // If there are no primary inputs the compiler is in WMO mode and builds one
    // SILModule for the entire module.
    auto SM = performSILGeneration(mod, SILOpts, true);
    std::deque<PostSILGenInputs> PSGIs;
    PSGIs.push_back(PostSILGenInputs{
        std::move(SM), llvm::none_of(mod->getFiles(), fileIsSIB), mod});
    return PSGIs;
  }
  // If there are primary source files, build a separate SILModule for
  // each source file, and run the remaining SILOpt-Serialize-IRGen-LLVM
  // once for each such input.
  if (auto *PrimaryFile = Instance.getPrimarySourceFile()) {
    auto SM = performSILGeneration(*PrimaryFile, SILOpts, None);
    std::deque<PostSILGenInputs> PSGIs;
    PSGIs.push_back(PostSILGenInputs{std::move(SM), true, PrimaryFile});
    return PSGIs;
  }
  // If there are primary inputs but no primary _source files_, there might be
  // a primary serialized input.
  std::deque<PostSILGenInputs> PSGIs;
  for (FileUnit *fileUnit : mod->getFiles()) {
    if (auto SASTF = dyn_cast<SerializedASTFile>(fileUnit))
      if (Invocation.getFrontendOptions().InputsAndOutputs.isInputPrimary(
              SASTF->getFilename())) {
        assert(PSGIs.empty() && "Can only handle one primary AST input");
        auto SM = performSILGeneration(*SASTF, SILOpts, None);
        PSGIs.push_back(
            PostSILGenInputs{std::move(SM), !fileIsSIB(SASTF), mod});
      }
  }
  return PSGIs;
}

static bool performCompileStepsPostSILGen(CompilerInstance &Instance,
                                          CompilerInvocation &Invocation,
                                          std::unique_ptr<SILModule> SM,
                                          bool astGuaranteedToCorrespondToSIL,
                                          ModuleOrSourceFile MSF,
                                          bool moduleIsPublic,
                                          int &ReturnValue,
                                          FrontendObserver *observer,
                                          UnifiedStatsReporter *Stats);

/// Performs the compile requested by the user.
/// \param Instance Will be reset after performIRGeneration when the verifier
///                 mode is NoVerify and there were no errors.
/// \returns true on error
static bool performCompile(CompilerInstance &Instance,
                           CompilerInvocation &Invocation,
                           ArrayRef<const char *> Args,
                           int &ReturnValue,
                           FrontendObserver *observer,
                           UnifiedStatsReporter *Stats) {
  FrontendOptions opts = Invocation.getFrontendOptions();
  FrontendOptions::ActionType Action = opts.RequestedAction;

  if (Action == FrontendOptions::ActionType::EmitSyntax) {
    Instance.getASTContext().LangOpts.KeepSyntaxInfoInSourceFile = true;
    Instance.getASTContext().LangOpts.VerifySyntaxTree = true;
  }

  // We've been asked to precompile a bridging header; we want to
  // avoid touching any other inputs and just parse, emit and exit.
  if (Action == FrontendOptions::ActionType::EmitPCH)
    return precompileBridgingHeader(Invocation, Instance);

    if (Invocation.getInputKind() == InputFileKind::IFK_LLVM_IR)
      return compileLLVMIR(Invocation, Instance, Stats);

  ReferencedNameTracker nameTracker;
  if (!opts.ReferenceDependenciesFilePath.empty())
    Instance.setReferencedNameTracker(&nameTracker);

  if (FrontendOptions::shouldActionOnlyParse(Action))
    Instance.performParseOnly();
  else
    Instance.performSema();

  if (Action == FrontendOptions::ActionType::Parse)
    return Instance.getASTContext().hadError();

  if (observer) {
    observer->performedSemanticAnalysis(Instance);
  }

  if (Stats) {
    countStatsPostSema(*Stats, Instance);
  }

  FrontendOptions::DebugCrashMode CrashMode = opts.CrashMode;
  if (CrashMode == FrontendOptions::DebugCrashMode::AssertAfterParse)
    debugFailWithAssertion();
  else if (CrashMode == FrontendOptions::DebugCrashMode::CrashAfterParse)
    debugFailWithCrash();

  ASTContext &Context = Instance.getASTContext();

  verifyGenericSignaturesIfNeeded(Invocation, Context);

  if (Invocation.getMigratorOptions().shouldRunMigrator()) {
    migrator::updateCodeAndEmitRemap(&Instance, Invocation);
  }

  if (Action == FrontendOptions::ActionType::REPL) {
    runREPL(Instance, ProcessCmdLine(Args.begin(), Args.end()),
            Invocation.getParseStdlib());
    return Context.hadError();
  }

  if (auto r = dumpASTIfNeeded(Invocation, Instance))
    return *r;

  // If we were asked to print Clang stats, do so.
  if (opts.PrintClangStats && Context.getClangModuleLoader())
    Context.getClangModuleLoader()->printStatistics();

  if (!opts.DependenciesFilePath.empty())
    (void)emitMakeDependencies(Context.Diags, *Instance.getDependencyTracker(),
                               opts);

  emitReferenceDependenciesIfNeeded(Invocation, Instance);

  (void)emitLoadedModuleTraceIfNeeded(Context, *Instance.getDependencyTracker(),
                                      opts);

  bool shouldIndex = !opts.IndexStorePath.empty();

  if (Context.hadError()) {
    if (shouldIndex) {
      //  Emit the index store data even if there were compiler errors.
      if (emitIndexData(Instance.getPrimarySourceFile(),
                        Invocation, Instance))
        return true;
    }
    return true;
  }

  // FIXME: This is still a lousy approximation of whether the module file will
  // be externally consumed.
  bool moduleIsPublic =
      !Instance.getMainModule()->hasEntryPoint() &&
      opts.ImplicitObjCHeaderPath.empty() &&
      !Context.LangOpts.EnableAppExtensionRestrictions;

  // We've just been told to perform a typecheck, so we can return now.
  if (Action == FrontendOptions::ActionType::Typecheck) {
    if (!opts.ObjCHeaderOutputPath.empty())
      return printAsObjC(opts.ObjCHeaderOutputPath, Instance.getMainModule(),
                         opts.ImplicitObjCHeaderPath, moduleIsPublic);
    if (shouldIndex) {
      if (emitIndexData(Instance.getPrimarySourceFile(),
                        Invocation, Instance))
        return true;
    }
    return Context.hadError();
  }

  if (writeTBDIfNeeded(Invocation, Instance))
    return true;

  assert(Action >= FrontendOptions::ActionType::EmitSILGen &&
         "All actions not requiring SILGen must have been handled!");

  std::deque<PostSILGenInputs> PSGIs = generateSILModules(Invocation, Instance);

  while (!PSGIs.empty()) {
    auto PSGI = std::move(PSGIs.front());
    PSGIs.pop_front();
    if (performCompileStepsPostSILGen(Instance, Invocation,
                                      std::move(PSGI.TheSILModule),
                                      PSGI.ASTGuaranteedToCorrespondToSIL,
                                      PSGI.ModuleOrPrimarySourceFile,
                                      moduleIsPublic,
                                      ReturnValue, observer, Stats))
      return true;
  }
  return false;
}

/// If we are asked to link all, link all.
static void linkAllIfNeeded(const CompilerInvocation &Invocation,
                            SILModule *SM) {
  if (Invocation.getSILOptions().LinkMode == SILOptions::LinkAll)
    performSILLinking(SM, true);
}

/// Perform "stable" optimizations that are invariant across compiler versions.
static bool performMandatorySILPasses(CompilerInvocation &Invocation,
                                      SILModule *SM,
                                      FrontendObserver *observer) {
  if (Invocation.getFrontendOptions().RequestedAction ==
      FrontendOptions::ActionType::MergeModules) {
    // Don't run diagnostic passes at all.
  } else if (!Invocation.getDiagnosticOptions().SkipDiagnosticPasses) {
    if (runSILDiagnosticPasses(*SM))
      return true;

    if (observer) {
      observer->performedSILDiagnostics(*SM);
    }
  } else {
    // Even if we are not supposed to run the diagnostic passes, we still need
    // to run the ownership evaluator.
    if (runSILOwnershipEliminatorPass(*SM))
      return true;
  }

  linkAllIfNeeded(Invocation, SM);

  if (Invocation.getSILOptions().MergePartialModules)
    SM->linkAllFromCurrentModule();
  return false;
}

static SerializationOptions
computeSerializationOptions(const CompilerInvocation &Invocation,
                            bool moduleIsPublic) {
  const FrontendOptions &opts = Invocation.getFrontendOptions();

  SerializationOptions serializationOpts;
  serializationOpts.OutputPath = opts.ModuleOutputPath.c_str();
  serializationOpts.DocOutputPath = opts.ModuleDocOutputPath.c_str();
  serializationOpts.GroupInfoPath = opts.GroupInfoPath.c_str();
  if (opts.SerializeBridgingHeader)
    serializationOpts.ImportedHeader = opts.ImplicitObjCHeaderPath;
  serializationOpts.ModuleLinkName = opts.ModuleLinkName;
  serializationOpts.ExtraClangOptions =
      Invocation.getClangImporterOptions().ExtraArgs;
  serializationOpts.EnableNestedTypeLookupTable =
      opts.EnableSerializationNestedTypeLookupTable;
  if (!Invocation.getIRGenOptions().ForceLoadSymbolName.empty())
    serializationOpts.AutolinkForceLoad = true;

  // Options contain information about the developer's computer,
  // so only serialize them if the module isn't going to be shipped to
  // the public.
  serializationOpts.SerializeOptionsForDebugging =
      !moduleIsPublic || opts.AlwaysSerializeDebuggingOptions;

  return serializationOpts;
}

/// Perform SIL optimization passes if optimizations haven't been disabled.
/// These may change across compiler versions.
static void performSILOptimizations(CompilerInvocation &Invocation,
                                    SILModule *SM) {
  SharedTimer timer("SIL optimization");
  if (Invocation.getFrontendOptions().RequestedAction ==
          FrontendOptions::ActionType::MergeModules ||
      !Invocation.getSILOptions().shouldOptimize()) {
    runSILPassesForOnone(*SM);
    return;
  }
  runSILOptPreparePasses(*SM);

  StringRef CustomPipelinePath =
      Invocation.getSILOptions().ExternalPassPipelineFilename;
  if (!CustomPipelinePath.empty()) {
    runSILOptimizationPassesWithFileSpecification(*SM, CustomPipelinePath);
  } else {
    runSILOptimizationPasses(*SM);
  }
}

/// Get the main source file's private discriminator and attach it to
/// the compile unit's flags.
static void setPrivateDiscriminatorIfNeeded(IRGenOptions &IRGenOpts,
                                            ModuleOrSourceFile MSF) {
  if (IRGenOpts.DebugInfoKind == IRGenDebugInfoKind::None ||
      !MSF.is<SourceFile *>())
    return;
  Identifier PD = MSF.get<SourceFile *>()->getPrivateDiscriminator();
  if (!PD.empty())
    IRGenOpts.DWARFDebugFlags += (" -private-discriminator " + PD.str()).str();
}

static bool serializeSIB(FrontendOptions &opts, SILModule *SM,
                         ASTContext &Context, ModuleOrSourceFile MSF) {
  const std::string &moduleOutputPath = opts.ModuleOutputPath;
  assert(!moduleOutputPath.empty() && "must have an output path");

  SerializationOptions serializationOpts;
  serializationOpts.OutputPath = moduleOutputPath.c_str();
  serializationOpts.SerializeAllSIL = true;
  serializationOpts.IsSIB = true;
  serialize(MSF, serializationOpts, SM);
  return Context.hadError();
}

/// Serialize the SILModule if it was not serialized yet.
static void ensureSILModuleIsSerialized(SILModule *SM) {
  if (!SM->isSerialized())
    SM->serialize();
}

static bool performCompileStepsPostSILGen(CompilerInstance &Instance,
                                          CompilerInvocation &Invocation,
                                          std::unique_ptr<SILModule> SM,
                                          bool astGuaranteedToCorrespondToSIL,
                                          ModuleOrSourceFile MSF,
                                          bool moduleIsPublic,
                                          int &ReturnValue,
                                          FrontendObserver *observer,
                                          UnifiedStatsReporter *Stats) {

  FrontendOptions opts = Invocation.getFrontendOptions();
  FrontendOptions::ActionType Action = opts.RequestedAction;
  ASTContext &Context = Instance.getASTContext();
  SILOptions &SILOpts = Invocation.getSILOptions();
  IRGenOptions &IRGenOpts = Invocation.getIRGenOptions();
  bool shouldIndex = !opts.IndexStorePath.empty();

  if (observer) {
    observer->performedSILGeneration(*SM);
  }
  if (Stats) {
    countStatsPostSILGen(*Stats, *SM);
  }

  // We've been told to emit SIL after SILGen, so write it now.
  if (Action == FrontendOptions::ActionType::EmitSILGen) {
    linkAllIfNeeded(Invocation, SM.get());
    return writeSIL(*SM, Instance, Invocation);
  }

  if (Action == FrontendOptions::ActionType::EmitSIBGen) {
    linkAllIfNeeded(Invocation, SM.get());
    serializeSIB(Invocation.getFrontendOptions(), SM.get(),
                 Instance.getASTContext(), MSF);
    return Context.hadError();
  }

  std::unique_ptr<llvm::raw_fd_ostream> OptRecordFile =
      createOptRecordFile(SILOpts.OptRecordFile, Instance.getDiags());
  if (OptRecordFile)
    SM->setOptRecordStream(llvm::make_unique<llvm::yaml::Output>(
                               *OptRecordFile, &Instance.getSourceMgr()),
                           std::move(OptRecordFile));

  if (performMandatorySILPasses(Invocation, SM.get(), observer))
    return true;

  {
    SharedTimer timer("SIL verification, pre-optimization");
    SM->verify();
  }

  // This is the action to be used to serialize SILModule.
  // It may be invoked multiple times, but it will perform
  // serialization only once. The serialization may either happen
  // after high-level optimizations or after all optimizations are
  // done, depending on the compiler setting.

  auto SerializeSILModuleAction = [&]() {
    if (opts.ModuleOutputPath.empty())
      return;

    SerializationOptions serializationOpts =
        computeSerializationOptions(Invocation, moduleIsPublic);
    serialize(MSF, serializationOpts, SM.get());
  };

  // Set the serialization action, so that the SIL module
  // can be serialized at any moment, e.g. during the optimization pipeline.
  SM->setSerializeSILAction(SerializeSILModuleAction);

  performSILOptimizations(Invocation, SM.get());

  if (observer) {
    observer->performedSILOptimization(*SM);
  }
  if (Stats) {
    countStatsPostSILOpt(*Stats, *SM);
  }

  {
    SharedTimer timer("SIL verification, post-optimization");
    SM->verify();
  }

  // Gather instruction counts if we are asked to do so.
  if (SM->getOptions().PrintInstCounts) {
    performSILInstCount(&*SM);
  }

  setPrivateDiscriminatorIfNeeded(IRGenOpts, MSF);

  if (!opts.ObjCHeaderOutputPath.empty()) {
    (void)printAsObjC(opts.ObjCHeaderOutputPath, Instance.getMainModule(),
                      opts.ImplicitObjCHeaderPath, moduleIsPublic);
  }

  if (Action == FrontendOptions::ActionType::EmitSIB)
    return serializeSIB(Invocation.getFrontendOptions(), SM.get(),
                        Instance.getASTContext(), MSF);

  if (!opts.ModuleOutputPath.empty() || !opts.ModuleDocOutputPath.empty()) {
    ensureSILModuleIsSerialized(SM.get());
    if (Action == FrontendOptions::ActionType::MergeModules ||
        Action == FrontendOptions::ActionType::EmitModuleOnly) {
      if (shouldIndex) {
        if (emitIndexData(MSF.dyn_cast<SourceFile*>(),
                          Invocation, Instance))
          return true;
      }
      return Context.hadError();
    }
  }

  assert(Action >= FrontendOptions::ActionType::EmitSIL &&
         "All actions not requiring SILPasses must have been handled!");

  // We've been told to write canonical SIL, so write it now.
  if (Action == FrontendOptions::ActionType::EmitSIL)
    return writeSIL(*SM, Instance, Invocation);

  assert(Action >= FrontendOptions::ActionType::Immediate &&
         "All actions not requiring IRGen must have been handled!");
  assert(Action != FrontendOptions::ActionType::REPL &&
         "REPL mode must be handled immediately after Instance->performSema()");
{'fraction_non_alphanumeric': 0.11354312274840044, 'fraction_numerical': 0.0005454979373359244, 'mean_word_length': 3.1481167208787064, 'pattern_counts': {'":': 2, '<': 171, '<?xml version=': 0, '>': 217, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 148, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4497325', 'n_tokens_mistral': 25420, 'n_tokens_neox': 25072, 'n_words': 5471}