<commit_before>//===--- DependencyGraph.cpp - Track intra-module dependencies ------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

#include "swift/Driver/DependencyGraph.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/YAMLParser.h"

using namespace swift;

namespace {
  using DependencyMaskTy = DependencyGraphImpl::DependencyMaskTy;
  enum class DependencyKind : DependencyMaskTy {
    Name = 1 << 0,
    Type = 1 << 1
  };

  enum class DependencyDirection : bool {
    Depends,
    Provides
  };
} // end anonymous namespace

static bool
parseDependencyFile(llvm::MemoryBuffer &buffer, bool providesOnly,
                    llvm::function_ref<void(StringRef, DependencyKind,
                                            DependencyDirection)> callback) {
  using namespace llvm;

  // FIXME: Switch to a format other than YAML.
  llvm::SourceMgr SM;
  yaml::Stream stream(buffer.getMemBufferRef(), SM);
  auto I = stream.begin();
  if (I == stream.end() || !I->getRoot())
    return true;

  auto *topLevelMap = dyn_cast<yaml::MappingNode>(I->getRoot());
  if (!topLevelMap)
    return !isa<yaml::NullNode>(I->getRoot());

  SmallString<64> scratch;
  // FIXME: LLVM's YAML support does incremental parsing in such a way that
  // for-range loops break.
  for (auto i = topLevelMap->begin(), e = topLevelMap->end(); i != e; ++i) {
    auto *key = cast<yaml::ScalarNode>(i->getKey());

    using KindPair = std::pair<DependencyKind, DependencyDirection>;

    KindPair dirAndKind = llvm::StringSwitch<KindPair>(key->getValue(scratch))
      .Case("top-level", std::make_pair(DependencyKind::Name,
                                        DependencyDirection::Depends))
      .Case("member-access", std::make_pair(DependencyKind::Type,
                                            DependencyDirection::Depends))
      .Case("provides", std::make_pair(DependencyKind::Name,
                                       DependencyDirection::Provides))
      .Case("nominals", std::make_pair(DependencyKind::Type,
                                       DependencyDirection::Provides));

    if (providesOnly && dirAndKind.second != DependencyDirection::Provides)
      continue;

    auto *entries = cast<yaml::SequenceNode>(i->getValue());
    for (const yaml::Node &rawEntry : *entries) {
      auto *entry = cast<yaml::ScalarNode>(&rawEntry);<commit_msg>[Driver] Handle null lists in the YAML dependencies file.<commit_after>//===--- DependencyGraph.cpp - Track intra-module dependencies ------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

#include "swift/Driver/DependencyGraph.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/YAMLParser.h"

using namespace swift;

namespace {
  using DependencyMaskTy = DependencyGraphImpl::DependencyMaskTy;
  enum class DependencyKind : DependencyMaskTy {
    Name = 1 << 0,
    Type = 1 << 1
  };

  enum class DependencyDirection : bool {
    Depends,
    Provides
  };
} // end anonymous namespace

static bool
parseDependencyFile(llvm::MemoryBuffer &buffer, bool providesOnly,
                    llvm::function_ref<void(StringRef, DependencyKind,
                                            DependencyDirection)> callback) {
  using namespace llvm;

  // FIXME: Switch to a format other than YAML.
  llvm::SourceMgr SM;
  yaml::Stream stream(buffer.getMemBufferRef(), SM);
  auto I = stream.begin();
  if (I == stream.end() || !I->getRoot())
    return true;

  auto *topLevelMap = dyn_cast<yaml::MappingNode>(I->getRoot());
  if (!topLevelMap)
    return !isa<yaml::NullNode>(I->getRoot());

  SmallString<64> scratch;
  // FIXME: LLVM's YAML support does incremental parsing in such a way that
  // for-range loops break.
  for (auto i = topLevelMap->begin(), e = topLevelMap->end(); i != e; ++i) {
    if (isa<yaml::NullNode>(i->getValue()))
      continue;

    auto *key = cast<yaml::ScalarNode>(i->getKey());

    using KindPair = std::pair<DependencyKind, DependencyDirection>;

    KindPair dirAndKind = llvm::StringSwitch<KindPair>(key->getValue(scratch))
      .Case("top-level", std::make_pair(DependencyKind::Name,
                                        DependencyDirection::Depends))
      .Case("member-access", std::make_pair(DependencyKind::Type,
                                            DependencyDirection::Depends))
      .Case("provides", std::make_pair(DependencyKind::Name,
                                       DependencyDirection::Provides))
      .Case("nominals", std::make_pair(DependencyKind::Type,
                                       DependencyDirection::Provides));

    if (providesOnly && dirAndKind.second != DependencyDirection::Provides)
      continue;

    auto *entries = cast<yaml::SequenceNode>(i->getValue());
    for (const yaml::Node &rawEntry : *entries) {
      auto *entry = cast<yaml::ScalarNode>(&rawEntry);
{'fraction_non_alphanumeric': 0.15867713378987808, 'fraction_numerical': 0.00534491398029063, 'mean_word_length': 2.7684078036500943, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5424875', 'n_tokens_mistral': 1735, 'n_tokens_neox': 1637, 'n_words': 406}