<commit_before>//===-- Frontend.cpp - frontend utility methods ---------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file contains utility methods for parsing and performing semantic
// on modules.
//
//===----------------------------------------------------------------------===//

#include "swift/Frontend/Frontend.h"
#include "swift/AST/ASTContext.h"
#include "swift/AST/Component.h"
#include "swift/AST/Diagnostics.h"
#include "swift/AST/Module.h"
#include "swift/ClangImporter/ClangImporter.h"
#include "swift/Parse/Lexer.h"
#include "swift/SIL/SILModule.h"
#include "swift/Serialization/SerializedModuleLoader.h"
#include "swift/Subsystems.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/SourceMgr.h"

using namespace swift;

/// \param SIL is non-null when we're parsing a .sil file instead of a .swift
/// file.
static TranslationUnit*
buildSingleTranslationUnit(ASTContext &Context,
                           StringRef ModuleName,
                           ArrayRef<unsigned> BufferIDs,
                           bool ParseOnly,
                           bool AllowBuiltinModule,
                           TranslationUnit::TUKind Kind,
                           SILModule *SIL,
                           std::unique_ptr<Parser> *TheParser) {
  Component *Comp = new (Context.Allocate<Component>(1)) Component();
  Identifier ID = Context.getIdentifier(ModuleName);
  TranslationUnit *TU = new (Context) TranslationUnit(ID, Comp, Context, Kind);
  Context.LoadedModules[ID.str()] = TU;

  TU->HasBuiltinModuleAccess = AllowBuiltinModule;

  // If we're in SIL mode, don't auto import any libraries.
  // Also don't perform auto import if we are not going to do semantic
  // analysis.
  if (Kind != TranslationUnit::SIL && !ParseOnly)
    performAutoImport(TU);

  if (Kind == TranslationUnit::Library) {
    // Parse all of the files into one big translation unit.
    for (auto &BufferID : BufferIDs) {
      bool Done;
      parseIntoTranslationUnit(TU, BufferID, &Done, nullptr, TheParser);
      assert(Done && "Parser returned early?");
      (void) Done;
      //performDelayedParsing(TU, TheParser->get());
      TheParser->reset(nullptr);
    }

    // Finally, if enabled, type check the whole thing in one go.
    if (!ParseOnly)
      performTypeChecking(TU);
    return TU;
  }

  // This may only be a main module or SIL, which requires pumping the parser.
  assert(Kind == TranslationUnit::Main || Kind == TranslationUnit::SIL);
  assert(BufferIDs.size() == 1 && "This mode only allows one input");
  unsigned BufferID = BufferIDs[0];

  SILParserState SILContext(SIL);

  unsigned CurTUElem = 0;
  bool Done;
  do {
    // Pump the parser multiple times if necessary.  It will return early
    // after parsing any top level code in a main module, or in SIL mode when
    // there are chunks of swift decls (e.g. imports and types) interspersed
    // with 'sil' definitions.
    parseIntoTranslationUnit(TU, BufferID, &Done,
                             SIL ? &SILContext : nullptr, TheParser);
    if (!ParseOnly)
      performTypeChecking(TU, CurTUElem);
    CurTUElem = TU->Decls.size();
  } while (!Done);

  return TU;
}

swift::CompilerInvocation::CompilerInvocation()
    : DriverDiagnostics(DriverDiagsSourceMgr) {
  TargetTriple = llvm::sys::getDefaultTargetTriple();
}

std::string swift::CompilerInvocation::getRuntimeIncludePath() const {
  llvm::SmallString<128> LibPath(MainExecutablePath);
  llvm::sys::path::remove_filename(LibPath); // Remove /swift
  llvm::sys::path::remove_filename(LibPath); // Remove /bin
  llvm::sys::path::append(LibPath, "lib", "swift");
  return LibPath.str();
}

void swift::CompilerInstance::createSILModule() {
  TheSILModule.reset(SILModule::createEmptyModule(getASTContext()));
}

void swift::CompilerInstance::setup() {
  for (auto DC : Invocation->getDiagnosticConsumers())
    Diagnostics.addConsumer(*DC);

  Context.reset(new ASTContext(Invocation->getLangOptions(), SourceMgr, Diagnostics));

  // Give the context the list of search paths to use for modules.
  Context->ImportSearchPaths = Invocation->getImportSearchPaths();
  Context->addModuleLoader(SourceLoader::create(*Context));
  Context->addModuleLoader(SerializedModuleLoader::create(*Context));

  // If the user has specified an SDK, wire up the Clang module importer
  // and point it at that SDK.
  if (!Invocation->getSDKPath().empty()) {
    auto clangImporter =
        ClangImporter::create(*Context, Invocation->getSDKPath(),
                              Invocation->getTargetTriple(),
                              Invocation->getClangModuleCachePath(),
                              Invocation->getImportSearchPaths());
    if (!clangImporter)
      return; // FIXME: error reporting

    Context->addModuleLoader(clangImporter, /*isClang*/true);
  }

  // Add the runtime include path (which contains swift.swift)
  Context->ImportSearchPaths.push_back(Invocation->getRuntimeIncludePath());

  assert(Lexer::isIdentifier(Invocation->getModuleName()));

  if (Invocation->getTUKind() == TranslationUnit::SIL)
    createSILModule();
}

void swift::CompilerInstance::doIt() {
  TU = buildSingleTranslationUnit(*Context,
                                  Invocation->getModuleName(),
                                  BufferIDs,
                                  Invocation->getParseOnly(),
                                  Invocation->getParseStdlib(),
                                  Invocation->getTUKind(),
                                  TheSILModule.get(),
                                  &TheParser);
}
<commit_msg>Inline buildSingleTranslationUnit() into its only caller<commit_after>//===-- Frontend.cpp - frontend utility methods ---------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file contains utility methods for parsing and performing semantic
// on modules.
//
//===----------------------------------------------------------------------===//

#include "swift/Frontend/Frontend.h"
#include "swift/AST/ASTContext.h"
#include "swift/AST/Component.h"
#include "swift/AST/Diagnostics.h"
#include "swift/AST/Module.h"
#include "swift/ClangImporter/ClangImporter.h"
#include "swift/Parse/Lexer.h"
#include "swift/SIL/SILModule.h"
#include "swift/Serialization/SerializedModuleLoader.h"
#include "swift/Subsystems.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/SourceMgr.h"

using namespace swift;

swift::CompilerInvocation::CompilerInvocation()
    : DriverDiagnostics(DriverDiagsSourceMgr) {
  TargetTriple = llvm::sys::getDefaultTargetTriple();
}

std::string swift::CompilerInvocation::getRuntimeIncludePath() const {
  llvm::SmallString<128> LibPath(MainExecutablePath);
  llvm::sys::path::remove_filename(LibPath); // Remove /swift
  llvm::sys::path::remove_filename(LibPath); // Remove /bin
  llvm::sys::path::append(LibPath, "lib", "swift");
  return LibPath.str();
}

void swift::CompilerInstance::createSILModule() {
  TheSILModule.reset(SILModule::createEmptyModule(getASTContext()));
}

void swift::CompilerInstance::setup() {
  for (auto DC : Invocation->getDiagnosticConsumers())
    Diagnostics.addConsumer(*DC);

  Context.reset(new ASTContext(Invocation->getLangOptions(), SourceMgr, Diagnostics));

  // Give the context the list of search paths to use for modules.
  Context->ImportSearchPaths = Invocation->getImportSearchPaths();
  Context->addModuleLoader(SourceLoader::create(*Context));
  Context->addModuleLoader(SerializedModuleLoader::create(*Context));

  // If the user has specified an SDK, wire up the Clang module importer
  // and point it at that SDK.
  if (!Invocation->getSDKPath().empty()) {
    auto clangImporter =
        ClangImporter::create(*Context, Invocation->getSDKPath(),
                              Invocation->getTargetTriple(),
                              Invocation->getClangModuleCachePath(),
                              Invocation->getImportSearchPaths());
    if (!clangImporter)
      return; // FIXME: error reporting

    Context->addModuleLoader(clangImporter, /*isClang*/true);
  }

  // Add the runtime include path (which contains swift.swift)
  Context->ImportSearchPaths.push_back(Invocation->getRuntimeIncludePath());

  assert(Lexer::isIdentifier(Invocation->getModuleName()));

  if (Invocation->getTUKind() == TranslationUnit::SIL)
    createSILModule();
}

void swift::CompilerInstance::doIt() {
  const TranslationUnit::TUKind Kind = Invocation->getTUKind();
  Component *Comp = new (Context->Allocate<Component>(1)) Component();
  Identifier ID = Context->getIdentifier(Invocation->getModuleName());
  TU = new (*Context) TranslationUnit(ID, Comp, *Context, Kind);
  Context->LoadedModules[ID.str()] = TU;

  TU->HasBuiltinModuleAccess = Invocation->getParseStdlib();

  // If we're in SIL mode, don't auto import any libraries.
  // Also don't perform auto import if we are not going to do semantic
  // analysis.
  if (Kind != TranslationUnit::SIL && !Invocation->getParseOnly())
    performAutoImport(TU);

  if (Kind == TranslationUnit::Library) {
    // Parse all of the files into one big translation unit.
    for (auto &BufferID : BufferIDs) {
      bool Done;
      parseIntoTranslationUnit(TU, BufferID, &Done, nullptr, &TheParser);
      assert(Done && "Parser returned early?");
      (void) Done;
      performDelayedParsing(TU, TheParser.get());
      TheParser.reset(nullptr);
    }

    // Finally, if enabled, type check the whole thing in one go.
    if (!Invocation->getParseOnly())
      performTypeChecking(TU);
    return;
  }

  // This may only be a main module or SIL, which requires pumping the parser.
  assert(Kind == TranslationUnit::Main || Kind == TranslationUnit::SIL);
  assert(BufferIDs.size() == 1 && "This mode only allows one input");
  unsigned BufferID = BufferIDs[0];

  SILParserState SILContext(TheSILModule.get());

  unsigned CurTUElem = 0;
  bool Done;
  do {
    // Pump the parser multiple times if necessary.  It will return early
    // after parsing any top level code in a main module, or in SIL mode when
    // there are chunks of swift decls (e.g. imports and types) interspersed
    // with 'sil' definitions.
    parseIntoTranslationUnit(TU, BufferID, &Done,
                             TheSILModule ? &SILContext : nullptr,
                             &TheParser);
    if (!Invocation->getParseOnly())
      performTypeChecking(TU, CurTUElem);
    CurTUElem = TU->Decls.size();
  } while (!Done);
}

{'fraction_non_alphanumeric': 0.15172595984501586, 'fraction_numerical': 0.0029940119760479044, 'mean_word_length': 2.931118033921772, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6479307', 'n_tokens_mistral': 3264, 'n_tokens_neox': 3104, 'n_words': 817}