<commit_before>// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/// An entrypoint used to run portions of front_end and measure its performance.
library front_end.tool.perf;

import 'dart:async';
import 'dart:io' show exit, stderr;

import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/error/listener.dart';
import 'package:analyzer/file_system/file_system.dart' show ResourceUriResolver;
import 'package:analyzer/file_system/physical_file_system.dart'
    show PhysicalResourceProvider;
import 'package:analyzer/source/package_map_resolver.dart';
import 'package:analyzer/src/context/builder.dart';
import 'package:analyzer/src/dart/sdk/sdk.dart' show FolderBasedDartSdk;
import 'package:analyzer/src/generated/parser.dart';
import 'package:analyzer/src/generated/source.dart';
import 'package:analyzer/src/generated/source_io.dart';
import 'package:kernel/analyzer/loader.dart';
import 'package:kernel/kernel.dart';
import 'package:package_config/discovery.dart';

import 'package:front_end/src/scanner/reader.dart';
import 'package:front_end/src/scanner/scanner.dart';
import 'package:front_end/src/scanner/token.dart';

/// Cumulative total number of chars scanned.
int scanTotalChars = 0;

/// Cumulative time spent scanning.
Stopwatch scanTimer = new Stopwatch();

/// Factory to load and resolve app, packages, and sdk sources.
SourceFactory sources;

main(List<String> args) async {
  // TODO(sigmund): provide sdk folder as well.
  if (args.length < 2) {
    print('usage: perf.dart <bench-id> <entry.dart>');
    exit(1);
  }
  var totalTimer = new Stopwatch()..start();

  var bench = args[0];
  var entryUri = Uri.base.resolve(args[1]);

  await setup(entryUri);

  var handlers = {
    'scan': () async {
      Set<Source> files = scanReachableFiles(entryUri);
      // TODO(sigmund): replace the warmup with instrumented snapshots.
      for (int i = 0; i < 10; i++) scanFiles(files);
    },
    'parse': () async {
      Set<Source> files = scanReachableFiles(entryUri);
      // TODO(sigmund): replace the warmup with instrumented snapshots.
      for (int i = 0; i < 10; i++) parseFiles(files);
    },
    'kernel_gen_e2e': () async {
      // TODO(sigmund): remove. This is used to compute the input size, we
      // should extract input size from frontend instead.
      scanReachableFiles(entryUri);
      // TODO(sigmund): replace this warmup. Note that for very large programs,
      // the GC pressure on the VM seems to make this worse with time (maybe we
      // are leaking memory?). That's why we run it twice and not 10 times.
      for (int i = 0; i < 2; i++) await generateKernel(entryUri);
    },
  };

  var handler = handlers[bench];
  if (handler == null) {
    // TODO(sigmund): implement the remaining benchmarks.
    print('unsupported bench-id: $bench. Please specify one of the following: '
        '${handlers.keys.join(", ")}');
    exit(1);
  }
  await handler();

  totalTimer.stop();
  report("total", totalTimer.elapsedMicroseconds);
}

/// Sets up analyzer to be able to load and resolve app, packages, and sdk
/// sources.
Future setup(Uri entryUri) async {
  var provider = PhysicalResourceProvider.INSTANCE;
  var packageMap = new ContextBuilder(provider, null, null)
      .convertPackagesToMap(await findPackages(entryUri));
  sources = new SourceFactory([
    new ResourceUriResolver(provider),
    new PackageMapUriResolver(provider, packageMap),
    new DartUriResolver(
        new FolderBasedDartSdk(provider, provider.getFolder("sdk"))),
  ]);
}

/// Load and scans all files we need to process: files reachable from the
/// entrypoint and all core libraries automatically included by the VM.
Set<Source> scanReachableFiles(Uri entryUri) {
  var files = new Set<Source>();
  var loadTimer = new Stopwatch()..start();
  collectSources(sources.forUri2(entryUri), files);

  var libs = [
    "dart:async",
    "dart:collection",
    "dart:convert",
    "dart:core",
    "dart:developer",
    "dart:_internal",
    "dart:isolate",
    "dart:math",
    "dart:mirrors",
    "dart:typed_data",
    "dart:io"
  ];

  for (var lib in libs) {
    collectSources(sources.forUri(lib), files);
  }

  loadTimer.stop();

  print('input size: ${scanTotalChars} chars');
  var loadTime = loadTimer.elapsedMicroseconds - scanTimer.elapsedMicroseconds;
  report("load", loadTime);
  report("scan", scanTimer.elapsedMicroseconds);
  return files;
}

/// Scans every file in [files] and reports the time spent doing so.
void scanFiles(Set<Source> files) {
  // The code below will record again how many chars are scanned and how long it
  // takes to scan them, even though we already did so in [scanReachableFiles].
  // Recording and reporting this twice is unnecessary, but we do so for now to
  // validate that the results are consistent.
  scanTimer = new Stopwatch();
  var old = scanTotalChars;
  scanTotalChars = 0;
  for (var source in files) {
    tokenize(source);
  }

  // Report size and scanning time again. See discussion above.
  if (old != scanTotalChars) print('input size changed? ${old} chars');
  report("scan", scanTimer.elapsedMicroseconds);
}

/// Parses every file in [files] and reports the time spent doing so.
void parseFiles(Set<Source> files) {
  // The code below will record again how many chars are scanned and how long it
  // takes to scan them, even though we already did so in [scanReachableFiles].
  // Recording and reporting this twice is unnecessary, but we do so for now to
  // validate that the results are consistent.
  scanTimer = new Stopwatch();
  var old = scanTotalChars;
  scanTotalChars = 0;
  var parseTimer = new Stopwatch()..start();
  for (var source in files) {
    parseFull(source);
  }
  parseTimer.stop();

  // Report size and scanning time again. See discussion above.
  if (old != scanTotalChars) print('input size changed? ${old} chars');
  report("scan", scanTimer.elapsedMicroseconds);

  var pTime = parseTimer.elapsedMicroseconds - scanTimer.elapsedMicroseconds;
  report("parse", pTime);
}

/// Add to [files] all sources reachable from [start].
void collectSources(Source start, Set<Source> files) {
  if (!files.add(start)) return;
  var unit = parseDirectives(start);
  for (var directive in unit.directives) {
    if (directive is UriBasedDirective) {
      var next = sources.resolveUri(start, directive.uri.stringValue);
      collectSources(next, files);
    }
  }
}

/// Uses the diet-parser to parse only directives in [source].
CompilationUnit parseDirectives(Source source) {
  var token = tokenize(source);
  var parser = new Parser(source, AnalysisErrorListener.NULL_LISTENER);
  return parser.parseDirectives(token);
}

/// Parse the full body of [source] and return it's compilation unit.
CompilationUnit parseFull(Source source) {
  var token = tokenize(source);
  var parser = new Parser(source, AnalysisErrorListener.NULL_LISTENER);
  return parser.parseCompilationUnit(token);
}<commit_msg>Add a front end perf test that computes unlinked summaries.<commit_after>// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/// An entrypoint used to run portions of front_end and measure its performance.
library front_end.tool.perf;

import 'dart:async';
import 'dart:io' show exit, stderr;

import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/error/listener.dart';
import 'package:analyzer/file_system/file_system.dart' show ResourceUriResolver;
import 'package:analyzer/file_system/physical_file_system.dart'
    show PhysicalResourceProvider;
import 'package:analyzer/source/package_map_resolver.dart';
import 'package:analyzer/src/context/builder.dart';
import 'package:analyzer/src/dart/sdk/sdk.dart' show FolderBasedDartSdk;
import 'package:analyzer/src/generated/parser.dart';
import 'package:analyzer/src/generated/source.dart';
import 'package:analyzer/src/generated/source_io.dart';
import 'package:analyzer/src/summary/format.dart';
import 'package:analyzer/src/summary/summarize_ast.dart';
import 'package:kernel/analyzer/loader.dart';
import 'package:kernel/kernel.dart';
import 'package:package_config/discovery.dart';

import 'package:front_end/src/scanner/reader.dart';
import 'package:front_end/src/scanner/scanner.dart';
import 'package:front_end/src/scanner/token.dart';

/// Cumulative total number of chars scanned.
int scanTotalChars = 0;

/// Cumulative time spent scanning.
Stopwatch scanTimer = new Stopwatch();

/// Factory to load and resolve app, packages, and sdk sources.
SourceFactory sources;

main(List<String> args) async {
  // TODO(sigmund): provide sdk folder as well.
  if (args.length < 2) {
    print('usage: perf.dart <bench-id> <entry.dart>');
    exit(1);
  }
  var totalTimer = new Stopwatch()..start();

  var bench = args[0];
  var entryUri = Uri.base.resolve(args[1]);

  await setup(entryUri);

  var handlers = {
    'scan': () async {
      Set<Source> files = scanReachableFiles(entryUri);
      // TODO(sigmund): replace the warmup with instrumented snapshots.
      for (int i = 0; i < 10; i++) scanFiles(files);
    },
    'parse': () async {
      Set<Source> files = scanReachableFiles(entryUri);
      // TODO(sigmund): replace the warmup with instrumented snapshots.
      for (int i = 0; i < 10; i++) parseFiles(files);
    },
    'kernel_gen_e2e': () async {
      // TODO(sigmund): remove. This is used to compute the input size, we
      // should extract input size from frontend instead.
      scanReachableFiles(entryUri);
      // TODO(sigmund): replace this warmup. Note that for very large programs,
      // the GC pressure on the VM seems to make this worse with time (maybe we
      // are leaking memory?). That's why we run it twice and not 10 times.
      for (int i = 0; i < 2; i++) await generateKernel(entryUri);
    },
    'unlinked_summarize': () async {
      Set<Source> files = scanReachableFiles(entryUri);
      // TODO(sigmund): replace the warmup with instrumented snapshots.
      for (int i = 0; i < 10; i++) unlinkedSummarizeFiles(files);
    }
  };

  var handler = handlers[bench];
  if (handler == null) {
    // TODO(sigmund): implement the remaining benchmarks.
    print('unsupported bench-id: $bench. Please specify one of the following: '
        '${handlers.keys.join(", ")}');
    exit(1);
  }
  await handler();

  totalTimer.stop();
  report("total", totalTimer.elapsedMicroseconds);
}

/// Sets up analyzer to be able to load and resolve app, packages, and sdk
/// sources.
Future setup(Uri entryUri) async {
  var provider = PhysicalResourceProvider.INSTANCE;
  var packageMap = new ContextBuilder(provider, null, null)
      .convertPackagesToMap(await findPackages(entryUri));
  sources = new SourceFactory([
    new ResourceUriResolver(provider),
    new PackageMapUriResolver(provider, packageMap),
    new DartUriResolver(
        new FolderBasedDartSdk(provider, provider.getFolder("sdk"))),
  ]);
}

/// Load and scans all files we need to process: files reachable from the
/// entrypoint and all core libraries automatically included by the VM.
Set<Source> scanReachableFiles(Uri entryUri) {
  var files = new Set<Source>();
  var loadTimer = new Stopwatch()..start();
  collectSources(sources.forUri2(entryUri), files);

  var libs = [
    "dart:async",
    "dart:collection",
    "dart:convert",
    "dart:core",
    "dart:developer",
    "dart:_internal",
    "dart:isolate",
    "dart:math",
    "dart:mirrors",
    "dart:typed_data",
    "dart:io"
  ];

  for (var lib in libs) {
    collectSources(sources.forUri(lib), files);
  }

  loadTimer.stop();

  print('input size: ${scanTotalChars} chars');
  var loadTime = loadTimer.elapsedMicroseconds - scanTimer.elapsedMicroseconds;
  report("load", loadTime);
  report("scan", scanTimer.elapsedMicroseconds);
  return files;
}

/// Scans every file in [files] and reports the time spent doing so.
void scanFiles(Set<Source> files) {
  // The code below will record again how many chars are scanned and how long it
  // takes to scan them, even though we already did so in [scanReachableFiles].
  // Recording and reporting this twice is unnecessary, but we do so for now to
  // validate that the results are consistent.
  scanTimer = new Stopwatch();
  var old = scanTotalChars;
  scanTotalChars = 0;
  for (var source in files) {
    tokenize(source);
  }

  // Report size and scanning time again. See discussion above.
  if (old != scanTotalChars) print('input size changed? ${old} chars');
  report("scan", scanTimer.elapsedMicroseconds);
}

/// Parses every file in [files] and reports the time spent doing so.
void parseFiles(Set<Source> files) {
  // The code below will record again how many chars are scanned and how long it
  // takes to scan them, even though we already did so in [scanReachableFiles].
  // Recording and reporting this twice is unnecessary, but we do so for now to
  // validate that the results are consistent.
  scanTimer = new Stopwatch();
  var old = scanTotalChars;
  scanTotalChars = 0;
  var parseTimer = new Stopwatch()..start();
  for (var source in files) {
    parseFull(source);
  }
  parseTimer.stop();

  // Report size and scanning time again. See discussion above.
  if (old != scanTotalChars) print('input size changed? ${old} chars');
  report("scan", scanTimer.elapsedMicroseconds);

  var pTime = parseTimer.elapsedMicroseconds - scanTimer.elapsedMicroseconds;
  report("parse", pTime);
}

/// Produces unlinked summaries for every file in [files] and reports the time
/// spent doing so.
void unlinkedSummarizeFiles(Set<Source> files) {
  // The code below will record again how many chars are scanned and how long it
  // takes to scan them, even though we already did so in [scanReachableFiles].
  // Recording and reporting this twice is unnecessary, but we do so for now to
  // validate that the results are consistent.
  scanTimer = new Stopwatch();
  var old = scanTotalChars;
  scanTotalChars = 0;
  var summarizeTimer = new Stopwatch()..start();
  for (var source in files) {
    unlinkedSummarize(source);
  }
  summarizeTimer.stop();

  if (old != scanTotalChars) print('input size changed? ${old} chars');

  // TODO(paulberry): subtract out scan/parse time?
  var summarizeTime = summarizeTimer.elapsedMicroseconds;
  report('unlinked summarize', summarizeTime);
}

/// Add to [files] all sources reachable from [start].
void collectSources(Source start, Set<Source> files) {
  if (!files.add(start)) return;
  var unit = parseDirectives(start);
  for (var directive in unit.directives) {
    if (directive is UriBasedDirective) {
      var next = sources.resolveUri(start, directive.uri.stringValue);
      collectSources(next, files);
    }
  }
}

/// Uses the diet-parser to parse only directives in [source].
CompilationUnit parseDirectives(Source source) {
  var token = tokenize(source);
  var parser = new Parser(source, AnalysisErrorListener.NULL_LISTENER);
  return parser.parseDirectives(token);
}

/// Parse the full body of [source] and return it's compilation unit.
CompilationUnit parseFull(Source source) {
  var token = tokenize(source);
  var parser = new Parser(source, AnalysisErrorListener.NULL_LISTENER);
  return parser.parseCompilationUnit(token);
}

UnlinkedUnitBuilder unlinkedSummarize(Source source) {
  var unit = parseFull(source);
  return serializeAstUnlinked(unit);
}
{'fraction_non_alphanumeric': 0.11505218003713426, 'fraction_numerical': 0.003329278442922082, 'mean_word_length': 3.984045947670708, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6173629', 'n_tokens_mistral': 4820, 'n_tokens_neox': 4659, 'n_words': 1516}