<commit_before>part of analysis;

class LibraryTuple {
  final CompilationUnit astUnit;

  final String path;
  final String packageUri;

  List<CompilationUnit> parts = [];

  List<CompilationUnit> get astUnits =>
    []..addAll(parts)..add(astUnit);

  String get name {
    String name;
    astUnit.directives
      .where((e) => e is LibraryDirective)
      .forEach((e) {
        name = e.name.name;
      });
    return name;
  }

  LibraryTuple._(this.astUnit, [this.path, this.packageUri]);
}

abstract class SourceCrawler implements Function {
  factory SourceCrawler({
      bool analyzeFunctionBodies: false,
      bool includeDefaultPackageRoot: true,
      List<String> packageRoots,
      bool preserveComments: false}) => new _SourceCrawler(analyzeFunctionBodies, includeDefaultPackageRoot, packageRoots, preserveComments);

  /// Resolves and crawls [path], and all files imported (recursively). Returns
  /// an [Iterable] of all the ASTs parsed.
  Iterable<LibraryTuple> call(String path);
  Iterable<LibraryTuple> crawl(String entryPointLocation, [bool deep = false]);
}

/// A source code crawler.
class _SourceCrawler implements SourceCrawler {
  final SourceResolver _sourceResolver;

  factory _SourceCrawler(
      bool analyzeFunctionBodies,
      bool includeDefaultPackageRoot,
      List<String> packageRoots,
      bool preserveComments) {
    if (packageRoots == null) {
      packageRoots = [Platform.packageRoot];
    }

    // Create a resolver to use.
    final sourceResolver = new SourceResolver.fromPackageRoots(
        packageRoots, includeDefaultPackageRoot);

    return new _SourceCrawler._(sourceResolver);
  }

  _SourceCrawler._(this._sourceResolver);

  /// Crawls, starting at [path], invoking [crawlFile] for each file.
  @override
  Iterable<LibraryTuple> call(String entryPointLocation) {
    return crawl(entryPointLocation);
  }

  Iterable<LibraryTuple> crawl(String entryPointLocation, [bool deep = false]) {
    final path = _getFileLocation(entryPointLocation);
    final astUnit = parseDartFile(path);

    var lib = new LibraryTuple._(astUnit, path);

    astUnit.directives
      .where((directive) => directive is PartDirective)
      .forEach((UriBasedDirective import) {
        final path = _getFileLocation(import.uri.stringValue, lib.path);
        if(path != null) {
          lib.parts.addAll(this.crawl(path, true).map((l) => l.astUnit));
        }
      });

    final results = <LibraryTuple>[];
    results.add(lib);

    astUnit.directives
      .where((directive) =>
        deep ? directive is ExportDirective :
        directive is ImportDirective || directive is ExportDirective)
      .forEach((UriBasedDirective import) {
        final path = _getFileLocation(import.uri.stringValue, lib.path);
        if(path != null) {
          results.addAll(this.crawl(path, true));
        }
      });

    return results;
  }

  String _getFileLocation(String uri, [String relativeTo]) {
    if (uri == null) {
      return null;
    } else if (uri.startsWith("package:") || uri.startsWith("dart:")) {
      return _sourceResolver(uri);
    } else {
      if(relativeTo != null)
        return path.join(path.dirname(relativeTo), uri);
      return path.absolute(uri);
    }
  }
}<commit_msg>Cache libraries.<commit_after>part of analysis;

class LibraryTuple {
  final CompilationUnit astUnit;

  final String path;
  final String packageUri;

  List<CompilationUnit> parts = [];

  List<CompilationUnit> get astUnits =>
    []..addAll(parts)..add(astUnit);

  String get name {
    String name;
    astUnit.directives
      .where((e) => e is LibraryDirective)
      .forEach((e) {
        name = e.name.name;
      });
    return name;
  }

  LibraryTuple._(this.astUnit, [this.path, this.packageUri]);
}

abstract class SourceCrawler implements Function {
  factory SourceCrawler({
      bool analyzeFunctionBodies: false,
      bool includeDefaultPackageRoot: true,
      List<String> packageRoots,
      bool preserveComments: false}) => new _SourceCrawler(analyzeFunctionBodies, includeDefaultPackageRoot, packageRoots, preserveComments);

  /// Resolves and crawls [path], and all files imported (recursively). Returns
  /// an [Iterable] of all the ASTs parsed.
  Iterable<LibraryTuple> call(String path);
  Iterable<LibraryTuple> crawl(String entryPointLocation, [bool deep = false]);
}

/// A source code crawler.
class _SourceCrawler implements SourceCrawler {
  final Map<String, LibraryTuple> _libraries = {};
  final SourceResolver _sourceResolver;

  factory _SourceCrawler(
      bool analyzeFunctionBodies,
      bool includeDefaultPackageRoot,
      List<String> packageRoots,
      bool preserveComments) {
    if (packageRoots == null) {
      packageRoots = [Platform.packageRoot];
    }

    // Create a resolver to use.
    final sourceResolver = new SourceResolver.fromPackageRoots(
        packageRoots, includeDefaultPackageRoot);

    return new _SourceCrawler._(sourceResolver);
  }

  _SourceCrawler._(this._sourceResolver);

  /// Crawls, starting at [path], invoking [crawlFile] for each file.
  @override
  Iterable<LibraryTuple> call(String entryPointLocation) {
    return crawl(entryPointLocation);
  }

  Iterable<LibraryTuple> crawl(String entryPointLocation, [bool deep = false]) {
    final path = _getFileLocation(entryPointLocation);
    final results = <LibraryTuple>[];

    if(_libraries.containsKey(path)) {
      results.add(_libraries[path]);
    } else {
      final astUnit = parseDartFile(path);
      var lib = new LibraryTuple._(astUnit, path);

      astUnit.directives
        .where((directive) => directive is PartDirective)
        .forEach((UriBasedDirective import) {
          final path = _getFileLocation(import.uri.stringValue, lib.path);
          if(path != null) {
            lib.parts.addAll(this.crawl(path, true).map((l) => l.astUnit));
          }
        });

      results.add(lib);
    }

    results.first.astUnit.directives
      .where((directive) =>
        deep ? directive is ExportDirective :
        directive is ImportDirective || directive is ExportDirective)
      .forEach((UriBasedDirective import) {
        final path = _getFileLocation(import.uri.stringValue, results.first.path);
        if(path != null) {
          results.addAll(this.crawl(path, true));
        }
      });

    return results;
  }

  String _getFileLocation(String uri, [String relativeTo]) {
    if (uri == null) {
      return null;
    } else if (uri.startsWith("package:") || uri.startsWith("dart:")) {
      return _sourceResolver(uri);
    } else {
      if(relativeTo != null)
        return path.join(path.dirname(relativeTo), uri);
      return path.absolute(uri);
    }
  }
}
{'fraction_non_alphanumeric': 0.11046598183712968, 'fraction_numerical': 0.0, 'mean_word_length': 3.2653968253968255, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3786228', 'n_tokens_mistral': 2011, 'n_tokens_neox': 1937, 'n_words': 464}