<commit_before>// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/**
 * This generates the reference documentation for the core libraries that come
 * with dart. It is built on top of dartdoc, which is a general-purpose library
 * for generating docs from any Dart code. This library extends that to include
 * additional information and styling specific to our standard library.
 *
 * Usage:
 *
 *     $ dart apidoc.dart [--out=<output directory>]
 */
library apidoc;

import 'dart:async';
import 'dart:io';
import 'dart:json' as json;

import 'html_diff.dart';

// TODO(rnystrom): Use "package:" URL (#4968).
import '../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors.dart';
import '../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors_util.dart';
import '../../sdk/lib/_internal/compiler/implementation/filenames.dart';
import '../../sdk/lib/_internal/dartdoc/lib/dartdoc.dart';
import '../../sdk/lib/_internal/libraries.dart';
import 'package:pathos/path.dart' as pathos;

HtmlDiff _diff;

void main() {
  final args = new Options().arguments;

  int mode = MODE_STATIC;
  Path outputDir = new Path('docs');
  bool generateAppCache = false;

  List<String> excludedLibraries = <String>[];

  // For libraries that have names matching the package name,
  // such as library unittest in package unittest, we just give
  // the package name with a --include-lib argument, such as:
  // --include-lib=unittest. These arguments are collected in
  // includedLibraries.
  List<String> includedLibraries = <String>[];

  // For libraries that lie within packages but have a different name,
  // such as the matcher library in package unittest, we can use 
  // --extra-lib with a full relative path under pkg, such as
  // --extra-lib=unittest/lib/matcher.dart. These arguments are
  // collected in extraLibraries.
  List<String> extraLibraries = <String>[];

  String packageRoot;
  String version;

  // Parse the command-line arguments.
  for (int i = 0; i < args.length; i++) {
    final arg = args[i];

    switch (arg) {
      case '--mode=static':
        mode = MODE_STATIC;
        break;

      case '--mode=live-nav':
        mode = MODE_LIVE_NAV;
        break;

      case '--generate-app-cache=true':
        generateAppCache = true;
        break;

      default:
        if (arg.startsWith('--exclude-lib=')) {
          excludedLibraries.add(arg.substring('--exclude-lib='.length));
        } else if (arg.startsWith('--include-lib=')) {
          includedLibraries.add(arg.substring('--include-lib='.length));
        } else if (arg.startsWith('--extra-lib=')) {
          extraLibraries.add(arg.substring('--extra-lib='.length));
        } else if (arg.startsWith('--out=')) {
          outputDir = new Path(arg.substring('--out='.length));
        } else if (arg.startsWith('--package-root=')) {
          packageRoot = arg.substring('--package-root='.length);
        } else if (arg.startsWith('--version=')) {
          version = arg.substring('--version='.length);
        } else {
          print('Unknown option: $arg');
          return;
        }
        break;
    }
  }

  final libPath = scriptDir.append('../../sdk/');

  cleanOutputDirectory(outputDir);

  print('Copying static files...');
  // The basic dartdoc-provided static content.
  final copiedStatic = copyDirectory(
      scriptDir.append('../../sdk/lib/_internal/dartdoc/static'),
      outputDir);

  // The apidoc-specific static content.
  final copiedApiDocStatic = copyDirectory(
      scriptDir.append('static'),
      outputDir);

  print('Parsing MDN data...');
  final mdnFile = new File.fromPath(scriptDir.append('mdn/database.json'));
  final mdn = json.parse(mdnFile.readAsStringSync());

  print('Cross-referencing dart:html...');
  // TODO(amouravski): move HtmlDiff inside of the future chain below to re-use
  // the MirrorSystem already analyzed.
  _diff = new HtmlDiff(printWarnings:false);
  Future htmlDiff = _diff.run(currentDirectory.resolve(libPath.toString()));

  // TODO(johnniwinther): Libraries for the compilation seem to be more like
  // URIs. Perhaps Path should have a toURI() method.
  // Add all of the core libraries.
  final apidocLibraries = <Uri>[];
  LIBRARIES.forEach((String name, LibraryInfo info) {
    if (info.documented) {
      apidocLibraries.add(Uri.parse('dart:$name'));
    }
  });

  // TODO(amouravski): This code is really wonky.
  var lister = new Directory.fromPath(scriptDir.append('../../pkg')).list();
  lister.listen((entity) {
    if (entity is Directory) {
      var path = new Path(entity.path);
      var libName = path.filename;
      var libPath = path.append('lib/$libName.dart');

      // Ignore some libraries.
      if (excludedLibraries.contains(libName)) {
        return;
      }

      // Ignore hidden directories (like .svn) as well as pkg.xcodeproj.
      if (libName.startsWith('.') || libName.endsWith('.xcodeproj')) {
        return;
      }

      if (new File.fromPath(libPath).existsSync()) {
        apidocLibraries.add(pathos.toUri(libPath.toNativePath()));
        includedLibraries.add(libName);
      } else {
        print('Warning: could not find package at $path');
      }
    }
  }, onDone: () {
    // Add any --extra libraries that had full pkg paths.
    // TODO(gram): if the handling of --include-lib libraries in the
    // listen() block above is cleaned up, then this will need to be
    // too, as it is a special case of the above.
    for (var lib in extraLibraries) {
      var libPath = new Path('../../$lib');
      if (new File.fromPath(libPath).existsSync()) {
        apidocLibraries.add(pathos.toUri(libPath.toNativePath()));
        var libName = libPath.filename.replaceAll('.dart', '');
        includedLibraries.add(libName);
      }
    }

    final apidoc = new Apidoc(mdn, outputDir, mode, generateAppCache,
                              excludedLibraries, version);
    apidoc.dartdocPath =
        scriptDir.append('../../sdk/lib/_internal/dartdoc/');
    // Select the libraries to include in the produced documentation:
    apidoc.includeApi = true;
    apidoc.includedLibraries = includedLibraries;

    // TODO(amouravski): make apidoc use roughly the same flow as bin/dartdoc.
    Future.wait([copiedStatic, copiedApiDocStatic, htmlDiff])
      .then((_) => apidoc.documentLibraries(apidocLibraries, libPath,
            packageRoot))
      .then((_) => compileScript(mode, outputDir, libPath, apidoc.tmpPath))
      .then((_) => print(apidoc.status))
      .catchError((e) {
        print('Error: generation failed: ${e}');
        var trace = getAttachedStackTrace(e);
        if (trace != null) print("StackTrace: $trace");
        apidoc.cleanup();
        exit(1);
      })
      .whenComplete(() => apidoc.cleanup());
  });
}

class Apidoc extends Dartdoc {
  /** Big ball of JSON containing the scraped MDN documentation. */
  final Map mdn;


  // A set of type names (TypeMirror.simpleName values) to ignore while
  // looking up information from MDN data.  TODO(eub, jacobr): fix up the MDN
  // import scripts so they run correctly and generate data that doesn't have
  // any entries that need to be ignored.
  static Set<String> _mdnTypeNamesToSkip = null;

  /**
   * The URL to the page on MDN that content was pulled from for the current
   * type being documented. Will be `null` if the type doesn't use any MDN
   * content.
   */
  String mdnUrl = null;

  Apidoc(this.mdn, Path outputDir, int mode,
         bool generateAppCache, [excludedLibraries, String version]) {
    if (?excludedLibraries) {
      this.excludedLibraries = excludedLibraries;
    }
    this.version = version;
    this.outputDir = outputDir;
    this.mode = mode;
    this.generateAppCache = generateAppCache;

    // Skip bad entries in the checked-in mdn/database.json:
    //  * UnknownElement has a top-level Gecko DOM page in German.
    if (_mdnTypeNamesToSkip == null)
      _mdnTypeNamesToSkip = new Set.from(['UnknownElement']);

    mainTitle = 'Dart API Reference';
    mainUrl = 'http://dartlang.org';

    final note    = 'http://code.google.com/policies.html#restrictions';
    final cca     = 'http://creativecommons.org/licenses/by/3.0/';
    final bsd     = 'http://code.google.com/google_bsd_license.html';
    final tos     = 'http://www.dartlang.org/tos.html';
    final privacy = 'http://www.google.com/intl/en/privacy/privacy-policy.html';

    footerText =
        '''
        <p>Except as otherwise <a href="$note">noted</a>, the content of this
        page is licensed under the <a href="$cca">Creative Commons Attribution
        3.0 License</a>, and code samples are licensed under the
        <a href="$bsd">BSD License</a>.</p>
        <p><a href="$tos">Terms of Service</a> |
        <a href="$privacy">Privacy Policy</a></p>
        ''';

    searchEngineId = '011220921317074318178:i4mscbaxtru';
    searchResultsUrl = 'http://www.dartlang.org/search.html';<commit_msg>Remove ? operator from Apidoc<commit_after>// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/**
 * This generates the reference documentation for the core libraries that come
 * with dart. It is built on top of dartdoc, which is a general-purpose library
 * for generating docs from any Dart code. This library extends that to include
 * additional information and styling specific to our standard library.
 *
 * Usage:
 *
 *     $ dart apidoc.dart [--out=<output directory>]
 */
library apidoc;

import 'dart:async';
import 'dart:io';
import 'dart:json' as json;

import 'html_diff.dart';

// TODO(rnystrom): Use "package:" URL (#4968).
import '../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors.dart';
import '../../sdk/lib/_internal/compiler/implementation/mirrors/mirrors_util.dart';
import '../../sdk/lib/_internal/compiler/implementation/filenames.dart';
import '../../sdk/lib/_internal/dartdoc/lib/dartdoc.dart';
import '../../sdk/lib/_internal/libraries.dart';
import 'package:pathos/path.dart' as pathos;

HtmlDiff _diff;

void main() {
  final args = new Options().arguments;

  int mode = MODE_STATIC;
  Path outputDir = new Path('docs');
  bool generateAppCache = false;

  List<String> excludedLibraries = <String>[];

  // For libraries that have names matching the package name,
  // such as library unittest in package unittest, we just give
  // the package name with a --include-lib argument, such as:
  // --include-lib=unittest. These arguments are collected in
  // includedLibraries.
  List<String> includedLibraries = <String>[];

  // For libraries that lie within packages but have a different name,
  // such as the matcher library in package unittest, we can use 
  // --extra-lib with a full relative path under pkg, such as
  // --extra-lib=unittest/lib/matcher.dart. These arguments are
  // collected in extraLibraries.
  List<String> extraLibraries = <String>[];

  String packageRoot;
  String version;

  // Parse the command-line arguments.
  for (int i = 0; i < args.length; i++) {
    final arg = args[i];

    switch (arg) {
      case '--mode=static':
        mode = MODE_STATIC;
        break;

      case '--mode=live-nav':
        mode = MODE_LIVE_NAV;
        break;

      case '--generate-app-cache=true':
        generateAppCache = true;
        break;

      default:
        if (arg.startsWith('--exclude-lib=')) {
          excludedLibraries.add(arg.substring('--exclude-lib='.length));
        } else if (arg.startsWith('--include-lib=')) {
          includedLibraries.add(arg.substring('--include-lib='.length));
        } else if (arg.startsWith('--extra-lib=')) {
          extraLibraries.add(arg.substring('--extra-lib='.length));
        } else if (arg.startsWith('--out=')) {
          outputDir = new Path(arg.substring('--out='.length));
        } else if (arg.startsWith('--package-root=')) {
          packageRoot = arg.substring('--package-root='.length);
        } else if (arg.startsWith('--version=')) {
          version = arg.substring('--version='.length);
        } else {
          print('Unknown option: $arg');
          return;
        }
        break;
    }
  }

  final libPath = scriptDir.append('../../sdk/');

  cleanOutputDirectory(outputDir);

  print('Copying static files...');
  // The basic dartdoc-provided static content.
  final copiedStatic = copyDirectory(
      scriptDir.append('../../sdk/lib/_internal/dartdoc/static'),
      outputDir);

  // The apidoc-specific static content.
  final copiedApiDocStatic = copyDirectory(
      scriptDir.append('static'),
      outputDir);

  print('Parsing MDN data...');
  final mdnFile = new File.fromPath(scriptDir.append('mdn/database.json'));
  final mdn = json.parse(mdnFile.readAsStringSync());

  print('Cross-referencing dart:html...');
  // TODO(amouravski): move HtmlDiff inside of the future chain below to re-use
  // the MirrorSystem already analyzed.
  _diff = new HtmlDiff(printWarnings:false);
  Future htmlDiff = _diff.run(currentDirectory.resolve(libPath.toString()));

  // TODO(johnniwinther): Libraries for the compilation seem to be more like
  // URIs. Perhaps Path should have a toURI() method.
  // Add all of the core libraries.
  final apidocLibraries = <Uri>[];
  LIBRARIES.forEach((String name, LibraryInfo info) {
    if (info.documented) {
      apidocLibraries.add(Uri.parse('dart:$name'));
    }
  });

  // TODO(amouravski): This code is really wonky.
  var lister = new Directory.fromPath(scriptDir.append('../../pkg')).list();
  lister.listen((entity) {
    if (entity is Directory) {
      var path = new Path(entity.path);
      var libName = path.filename;
      var libPath = path.append('lib/$libName.dart');

      // Ignore some libraries.
      if (excludedLibraries.contains(libName)) {
        return;
      }

      // Ignore hidden directories (like .svn) as well as pkg.xcodeproj.
      if (libName.startsWith('.') || libName.endsWith('.xcodeproj')) {
        return;
      }

      if (new File.fromPath(libPath).existsSync()) {
        apidocLibraries.add(pathos.toUri(libPath.toNativePath()));
        includedLibraries.add(libName);
      } else {
        print('Warning: could not find package at $path');
      }
    }
  }, onDone: () {
    // Add any --extra libraries that had full pkg paths.
    // TODO(gram): if the handling of --include-lib libraries in the
    // listen() block above is cleaned up, then this will need to be
    // too, as it is a special case of the above.
    for (var lib in extraLibraries) {
      var libPath = new Path('../../$lib');
      if (new File.fromPath(libPath).existsSync()) {
        apidocLibraries.add(pathos.toUri(libPath.toNativePath()));
        var libName = libPath.filename.replaceAll('.dart', '');
        includedLibraries.add(libName);
      }
    }

    final apidoc = new Apidoc(mdn, outputDir, mode, generateAppCache,
                              excludedLibraries, version);
    apidoc.dartdocPath =
        scriptDir.append('../../sdk/lib/_internal/dartdoc/');
    // Select the libraries to include in the produced documentation:
    apidoc.includeApi = true;
    apidoc.includedLibraries = includedLibraries;

    // TODO(amouravski): make apidoc use roughly the same flow as bin/dartdoc.
    Future.wait([copiedStatic, copiedApiDocStatic, htmlDiff])
      .then((_) => apidoc.documentLibraries(apidocLibraries, libPath,
            packageRoot))
      .then((_) => compileScript(mode, outputDir, libPath, apidoc.tmpPath))
      .then((_) => print(apidoc.status))
      .catchError((e) {
        print('Error: generation failed: ${e}');
        var trace = getAttachedStackTrace(e);
        if (trace != null) print("StackTrace: $trace");
        apidoc.cleanup();
        exit(1);
      })
      .whenComplete(() => apidoc.cleanup());
  });
}

class Apidoc extends Dartdoc {
  /** Big ball of JSON containing the scraped MDN documentation. */
  final Map mdn;


  // A set of type names (TypeMirror.simpleName values) to ignore while
  // looking up information from MDN data.  TODO(eub, jacobr): fix up the MDN
  // import scripts so they run correctly and generate data that doesn't have
  // any entries that need to be ignored.
  static Set<String> _mdnTypeNamesToSkip = null;

  /**
   * The URL to the page on MDN that content was pulled from for the current
   * type being documented. Will be `null` if the type doesn't use any MDN
   * content.
   */
  String mdnUrl = null;

  Apidoc(this.mdn, Path outputDir, int mode, bool generateAppCache,
      [List<String> excludedLibraries, String version]) {
    if (excludedLibraries != null) this.excludedLibraries = excludedLibraries;
    this.version = version;
    this.outputDir = outputDir;
    this.mode = mode;
    this.generateAppCache = generateAppCache;

    // Skip bad entries in the checked-in mdn/database.json:
    //  * UnknownElement has a top-level Gecko DOM page in German.
    if (_mdnTypeNamesToSkip == null)
      _mdnTypeNamesToSkip = new Set.from(['UnknownElement']);

    mainTitle = 'Dart API Reference';
    mainUrl = 'http://dartlang.org';

    final note    = 'http://code.google.com/policies.html#restrictions';
    final cca     = 'http://creativecommons.org/licenses/by/3.0/';
    final bsd     = 'http://code.google.com/google_bsd_license.html';
    final tos     = 'http://www.dartlang.org/tos.html';
    final privacy = 'http://www.google.com/intl/en/privacy/privacy-policy.html';

    footerText =
        '''
        <p>Except as otherwise <a href="$note">noted</a>, the content of this
        page is licensed under the <a href="$cca">Creative Commons Attribution
        3.0 License</a>, and code samples are licensed under the
        <a href="$bsd">BSD License</a>.</p>
        <p><a href="$tos">Terms of Service</a> |
        <a href="$privacy">Privacy Policy</a></p>
        ''';

    searchEngineId = '011220921317074318178:i4mscbaxtru';
    searchResultsUrl = 'http://www.dartlang.org/search.html';
{'fraction_non_alphanumeric': 0.13592233009708737, 'fraction_numerical': 0.003949317097251934, 'mean_word_length': 2.8865913451289704, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4706190', 'n_tokens_mistral': 5637, 'n_tokens_neox': 5466, 'n_words': 1530}