<commit_before>//===--- ASTSectionImporter.cpp - Import AST Section Modules --------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file implements support for loading modules serialized into a
// Mach-O AST section into Swift.
//
//===----------------------------------------------------------------------===//

#include "swift/Serialization/SerializedModuleLoader.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"

namespace swift {

bool parseASTSection(SerializedModuleLoader* SML, StringRef Buf,
                     SmallVectorImpl<std::string> &foundModules) {
  struct apple_ast_hdr {
    uint32_t version;
    uint32_t nmods;
  };

  struct module_header {
   uint64_t bitstream_ofs;
   uint64_t bitstream_size;
   uint64_t name_ofs;
   uint32_t language;
   uint32_t flags;
  };

  size_t size = Buf.size();
  const char *data = Buf.data();

  if (size < sizeof(struct apple_ast_hdr) + sizeof(struct module_header)) {
    llvm::dbgs() << "__ast section is too small.\n";
    return false;
  }

  auto apple_ast_hdr = reinterpret_cast<const struct apple_ast_hdr *>(data);
  if (apple_ast_hdr->version != 1) {
    llvm::dbgs() << "Unsupported __ast section version.\n";
    return false;
  }

  // Iterate over all AST modules.
  for (uint32_t i = 0; i < apple_ast_hdr->nmods; ++i) {
    auto mh = reinterpret_cast<const struct module_header *>
      (data+sizeof(apple_ast_hdr));

    if (mh->language != dwarf::DW_LANG_Swift)
      continue;

    // Get the access path.
    if (mh->name_ofs + 4 > size) return false;
    auto nchars = *reinterpret_cast<const uint32_t *>(data + mh->name_ofs);
    if (mh->name_ofs+sizeof(nchars) > size) return false;
    assert(nchars < (2 << 10) && "path failed sanity check");
    llvm::StringRef AccessPath(data+mh->name_ofs+sizeof(nchars), nchars);

    // loadModule() wants to take ownership of the input memory
    // buffer, but we don't let it own the memory, since it's just a
    // window into Buf.
    if (mh->bitstream_ofs + mh->bitstream_size > size) return false;
    auto mem = llvm::StringRef(data+mh->bitstream_ofs, mh->bitstream_size);
    auto bitstream = llvm::MemoryBuffer::getMemBuffer(mem, AccessPath);

    // Register the memory buffer.
    SML->registerMemoryBuffer(AccessPath,
                              std::unique_ptr<llvm::MemoryBuffer>(bitstream));
    foundModules.push_back(AccessPath);
  }
  return true;
}

}<commit_msg>Debug info: Implement multi-module swift ASTSections. When multiple swift .o files are linked, ld automatically concatenates the swift/AST sections. Updated section header, SwiftASTstreamer, ASTSectionImporter, and lldb-moduleimport-test to support this feature.<commit_after>//===--- ASTSectionImporter.cpp - Import AST Section Modules --------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file implements support for loading modules serialized into a
// Mach-O AST section into Swift.
//
//===----------------------------------------------------------------------===//

#include "swift/Serialization/SerializedModuleLoader.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"

namespace swift {

bool parseASTSection(SerializedModuleLoader* SML, StringRef Buf,
                     SmallVectorImpl<std::string> &foundModules) {
  struct apple_ast_hdr {
    uint32_t magic;
    uint32_t version;
    uint32_t language;
    uint32_t flags;
    uint64_t bitstream_ofs;
    uint64_t bitstream_size;
    uint32_t name_len;
    char name[0];
  };

  size_t size = Buf.size();
  const char *data = Buf.data();

  // An AST section consists of one or more AST modules + headers.
  // Iterate over all AST modules.
  while (size > sizeof(struct apple_ast_hdr)) {
    auto h = reinterpret_cast<const struct apple_ast_hdr *>(data);

    // Check for fatal errors first.
    if (h->magic != 0x41535473) {
      llvm::dbgs() << "Magic number not found.\n";
      return false;
    }

    if (h->version != 1) {
      llvm::dbgs() << "Unsupported __ast section version.\n";
      return false;
    }
    
    if (h->language != dwarf::DW_LANG_Swift)
      continue;

    // Get the access path.
    if (sizeof(h)+h->name_len > size) {
      llvm::dbgs() << "Impossible access path length. Section corrupted?\n";
      return false;
    }
    assert(h->name_len < (2 << 10) && "path failed sanity check");
    llvm::StringRef AccessPath(h->name, h->name_len);

    // loadModule() wants to take ownership of the input memory
    // buffer, but we don't let it own the memory, since it's just a
    // window into Buf.
    if (h->bitstream_ofs + h->bitstream_size > size) return false;
    auto mem = llvm::StringRef(data+h->bitstream_ofs, h->bitstream_size);
    auto bitstream = llvm::MemoryBuffer::getMemBuffer(mem, AccessPath);

    // Register the memory buffer.
    SML->registerMemoryBuffer(AccessPath,
                              std::unique_ptr<llvm::MemoryBuffer>(bitstream));
    foundModules.push_back(AccessPath);

    // Forward to the next module.
    uint64_t skip =
      llvm::RoundUpToAlignment(h->bitstream_ofs + h->bitstream_size, 32);

    if (skip > size) {
      llvm::dbgs() << "AST section too small.\n";
      return false;
    }

    data += skip;
    size -= skip;
  }
  return true;
}

}
{'fraction_non_alphanumeric': 0.16932697054467666, 'fraction_numerical': 0.012506170807964456, 'mean_word_length': 2.5879574970484063, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3216660', 'n_tokens_mistral': 1894, 'n_tokens_neox': 1775, 'n_words': 525}