<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "sky/tools/sky_snapshot/loader.h"

#include <memory>

#include "base/command_line.h"
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/logging.h"
#include "base/strings/string_util.h"
#include "sky/tools/sky_snapshot/logging.h"
#include "sky/tools/sky_snapshot/scope.h"
#include "sky/tools/sky_snapshot/switches.h"
#include "sky/engine/tonic/parsers/packages_map.h"

namespace {

// Extract the scheme prefix ('package:' or 'file:' from )
static std::string ExtractSchemePrefix(std::string url) {
  if (base::StartsWithASCII(url, "package:", true)) {
    return "package:";
  } else if (base::StartsWithASCII(url, "file:", true)) {
    return "file:";
  }
  return "";
}

// Extract the path from a package: or file: url.
static std::string ExtractPath(std::string url) {
  if (base::StartsWithASCII(url, "package:", true)) {
    base::ReplaceFirstSubstringAfterOffset(&url, 0, "package:", "");
  } else if (base::StartsWithASCII(url, "file:", true)) {
    base::ReplaceFirstSubstringAfterOffset(&url, 0, "file:", "");
  }
  return url;
}

base::FilePath SimplifyPath(const base::FilePath& path) {
  std::vector<base::FilePath::StringType> components;
  path.GetComponents(&components);
  auto it = components.begin();
  base::FilePath result(*it++);
  for (; it != components.end(); it++) {
    auto& component = *it;
    if (component == base::FilePath::kCurrentDirectory)
      continue;
    if (component == base::FilePath::kParentDirectory)
      result = result.DirName();
    else
      result = result.Append(component);
  }
  return result;
}

class Loader {
 public:
  Loader();

  void LoadPackagesMap(const base::FilePath& packages);

  const std::set<std::string>& dependencies() const { return dependencies_; }

  Dart_Handle CanonicalizeURL(Dart_Handle library, Dart_Handle url);
  std::string GetFilePathForURL(std::string url);
  std::string GetFilePathForPackageURL(std::string url);
  std::string GetFilePathForFileURL(std::string url);
  std::string Fetch(const std::string& url);
  Dart_Handle Import(Dart_Handle url);
  Dart_Handle Source(Dart_Handle library, Dart_Handle url);

  void set_package_root(const base::FilePath& package_root) {
    package_root_ = package_root;
  }

 private:
  std::set<std::string> dependencies_;
  base::FilePath packages_;
  base::FilePath package_root_;
  std::unique_ptr<tonic::PackagesMap> packages_map_;

  DISALLOW_COPY_AND_ASSIGN(Loader);
};

Loader::Loader() {
}

void Loader::LoadPackagesMap(const base::FilePath& packages) {
  packages_ = base::MakeAbsoluteFilePath(packages);
  dependencies_.insert(packages_.AsUTF8Unsafe());
  std::string packages_source;
  if (!base::ReadFileToString(packages_, &packages_source)) {
    fprintf(stderr, "error: Unable to load .packages file '%s'.\n",
        packages_.AsUTF8Unsafe().c_str());
    exit(1);
  }
  packages_map_.reset(new tonic::PackagesMap());
  std::string error;
  if (!packages_map_->Parse(packages_source, &error)) {
    fprintf(stderr, "error: Unable to parse .packages file '%s'.\n%s\n",
        packages_.AsUTF8Unsafe().c_str(), error.c_str());
    exit(1);
  }
}

Dart_Handle Loader::CanonicalizeURL(Dart_Handle library, Dart_Handle url) {
  std::string string = StringFromDart(url);
  if (base::StartsWithASCII(string, "dart:", true))
    return url;
  if (base::StartsWithASCII(string, "package:", true))
    return url;
  if (base::StartsWithASCII(string, "file:", true)) {
    base::ReplaceFirstSubstringAfterOffset(&string, 0, "file:", "");
    return StringToDart(string);;
  }

  std::string library_url = StringFromDart(Dart_LibraryUrl(library));
  std::string prefix = ExtractSchemePrefix(library_url);
  std::string path = ExtractPath(library_url);
  base::FilePath base_path(path);
  base::FilePath resolved_path = base_path.DirName().Append(string);
  base::FilePath normalized_path = SimplifyPath(resolved_path);
  return StringToDart(prefix + normalized_path.AsUTF8Unsafe());
}

std::string Loader::GetFilePathForURL(std::string url) {
  if (base::StartsWithASCII(url, "package:", true))
    return GetFilePathForPackageURL(url);
  if (base::StartsWithASCII(url, "file:", true))
    return GetFilePathForFileURL(url);

  return url;
}

std::string Loader::GetFilePathForPackageURL(
    std::string url) {
  DCHECK(base::StartsWithASCII(url, "package:", true));
  base::ReplaceFirstSubstringAfterOffset(&url, 0, "package:", "");
  size_t slash = url.find('/');
  if (slash == std::string::npos)
    return std::string();
  std::string package = url.substr(0, slash);
  std::string library_path = url.substr(slash + 1);
  std::string package_path = packages_map_->Resolve(package);
  if (package_path.empty())
    return std::string();
  if (base::StartsWithASCII(package_path, "file://", true)) {
    base::ReplaceFirstSubstringAfterOffset(&package_path, 0, "file://", "");
    return package_path + library_path;
  }
  auto path = packages_.DirName().Append(package_path).Append(library_path);
  return SimplifyPath(path).AsUTF8Unsafe();
}

std::string Loader::GetFilePathForFileURL(std::string url) {
  DCHECK(base::StartsWithASCII(url, "file://", true));
  base::ReplaceFirstSubstringAfterOffset(&url, 0, "file://", "");
  return url;
}

std::string Loader::Fetch(const std::string& url) {
  base::FilePath path(GetFilePathForURL(url));
  std::string source;
  if (!base::ReadFileToString(path, &source)) {
    fprintf(stderr, "error: Unable to find Dart library '%s'.\n", url.c_str());
    exit(1);
  }
  dependencies_.insert(url);
  return source;<commit_msg>Fix snapshotter deps file output (#2593)<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "sky/tools/sky_snapshot/loader.h"

#include <memory>

#include "base/command_line.h"
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/logging.h"
#include "base/strings/string_util.h"
#include "sky/tools/sky_snapshot/logging.h"
#include "sky/tools/sky_snapshot/scope.h"
#include "sky/tools/sky_snapshot/switches.h"
#include "sky/engine/tonic/parsers/packages_map.h"

namespace {

// Extract the scheme prefix ('package:' or 'file:' from )
static std::string ExtractSchemePrefix(std::string url) {
  if (base::StartsWithASCII(url, "package:", true)) {
    return "package:";
  } else if (base::StartsWithASCII(url, "file:", true)) {
    return "file:";
  }
  return "";
}

// Extract the path from a package: or file: url.
static std::string ExtractPath(std::string url) {
  if (base::StartsWithASCII(url, "package:", true)) {
    base::ReplaceFirstSubstringAfterOffset(&url, 0, "package:", "");
  } else if (base::StartsWithASCII(url, "file:", true)) {
    base::ReplaceFirstSubstringAfterOffset(&url, 0, "file:", "");
  }
  return url;
}

base::FilePath SimplifyPath(const base::FilePath& path) {
  std::vector<base::FilePath::StringType> components;
  path.GetComponents(&components);
  auto it = components.begin();
  base::FilePath result(*it++);
  for (; it != components.end(); it++) {
    auto& component = *it;
    if (component == base::FilePath::kCurrentDirectory)
      continue;
    if (component == base::FilePath::kParentDirectory)
      result = result.DirName();
    else
      result = result.Append(component);
  }
  return result;
}

class Loader {
 public:
  Loader();

  void LoadPackagesMap(const base::FilePath& packages);

  const std::set<std::string>& dependencies() const { return dependencies_; }

  Dart_Handle CanonicalizeURL(Dart_Handle library, Dart_Handle url);
  std::string GetFilePathForURL(std::string url);
  std::string GetFilePathForPackageURL(std::string url);
  std::string GetFilePathForFileURL(std::string url);
  std::string Fetch(const std::string& url);
  Dart_Handle Import(Dart_Handle url);
  Dart_Handle Source(Dart_Handle library, Dart_Handle url);

  void set_package_root(const base::FilePath& package_root) {
    package_root_ = package_root;
  }

 private:
  std::set<std::string> dependencies_;
  base::FilePath packages_;
  base::FilePath package_root_;
  std::unique_ptr<tonic::PackagesMap> packages_map_;

  DISALLOW_COPY_AND_ASSIGN(Loader);
};

Loader::Loader() {
}

void Loader::LoadPackagesMap(const base::FilePath& packages) {
  packages_ = base::MakeAbsoluteFilePath(packages);
  dependencies_.insert(packages_.AsUTF8Unsafe());
  std::string packages_source;
  if (!base::ReadFileToString(packages_, &packages_source)) {
    fprintf(stderr, "error: Unable to load .packages file '%s'.\n",
        packages_.AsUTF8Unsafe().c_str());
    exit(1);
  }
  packages_map_.reset(new tonic::PackagesMap());
  std::string error;
  if (!packages_map_->Parse(packages_source, &error)) {
    fprintf(stderr, "error: Unable to parse .packages file '%s'.\n%s\n",
        packages_.AsUTF8Unsafe().c_str(), error.c_str());
    exit(1);
  }
}

Dart_Handle Loader::CanonicalizeURL(Dart_Handle library, Dart_Handle url) {
  std::string string = StringFromDart(url);
  if (base::StartsWithASCII(string, "dart:", true))
    return url;
  if (base::StartsWithASCII(string, "package:", true))
    return url;
  if (base::StartsWithASCII(string, "file:", true)) {
    base::ReplaceFirstSubstringAfterOffset(&string, 0, "file:", "");
    return StringToDart(string);;
  }

  std::string library_url = StringFromDart(Dart_LibraryUrl(library));
  std::string prefix = ExtractSchemePrefix(library_url);
  std::string path = ExtractPath(library_url);
  base::FilePath base_path(path);
  base::FilePath resolved_path = base_path.DirName().Append(string);
  base::FilePath normalized_path = SimplifyPath(resolved_path);
  return StringToDart(prefix + normalized_path.AsUTF8Unsafe());
}

std::string Loader::GetFilePathForURL(std::string url) {
  if (base::StartsWithASCII(url, "package:", true))
    return GetFilePathForPackageURL(url);
  if (base::StartsWithASCII(url, "file:", true))
    return GetFilePathForFileURL(url);

  return url;
}

std::string Loader::GetFilePathForPackageURL(
    std::string url) {
  DCHECK(base::StartsWithASCII(url, "package:", true));
  base::ReplaceFirstSubstringAfterOffset(&url, 0, "package:", "");
  size_t slash = url.find('/');
  if (slash == std::string::npos)
    return std::string();
  std::string package = url.substr(0, slash);
  std::string library_path = url.substr(slash + 1);
  std::string package_path = packages_map_->Resolve(package);
  if (package_path.empty())
    return std::string();
  if (base::StartsWithASCII(package_path, "file://", true)) {
    base::ReplaceFirstSubstringAfterOffset(&package_path, 0, "file://", "");
    return package_path + library_path;
  }
  auto path = packages_.DirName().Append(package_path).Append(library_path);
  return SimplifyPath(path).AsUTF8Unsafe();
}

std::string Loader::GetFilePathForFileURL(std::string url) {
  DCHECK(base::StartsWithASCII(url, "file://", true));
  base::ReplaceFirstSubstringAfterOffset(&url, 0, "file://", "");
  return url;
}

std::string Loader::Fetch(const std::string& url) {
  std::string url_path = GetFilePathForURL(url);
  base::FilePath path(url_path);
  std::string source;
  if (!base::ReadFileToString(path, &source)) {
    fprintf(stderr, "error: Unable to find Dart library '%s'.\n", url.c_str());
    exit(1);
  }
  dependencies_.insert(url_path);
  return source;
{'fraction_non_alphanumeric': 0.15530862053997743, 'fraction_numerical': 0.0038197760222241513, 'mean_word_length': 5.501128668171558, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3930277', 'n_tokens_mistral': 3749, 'n_tokens_neox': 3717, 'n_words': 779}