<commit_before>import 'package:args/args.dart';
import 'package:crypto/crypto.dart' as crypto;
import 'package:dartdoc/src/model/model.dart';
import 'package:dartdoc/src/render/model_element_renderer.dart';
import 'package:dartdoc/src/utils.dart';
import 'package:dartdoc/src/warnings.dart';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path show Context;

final _templatePattern = RegExp(
    r'[ ]*{@template\s+(.+?)}([\s\S]+?){@endtemplate}[ ]*(\n?)',
    multiLine: true);
final _htmlPattern = RegExp(
    r'[ ]*{@inject-html\s*}([\s\S]+?){@end-inject-html}[ ]*\n?',
    multiLine: true);

/// Matches all tool directives (even some invalid ones). This is so
/// we can give good error messages if the directive is malformed, instead of
/// just silently emitting it as-is.
final _basicToolPattern = RegExp(
    r'[ ]*{@tool\s+([^}]+)}\n?([\s\S]+?)\n?{@end-tool}[ ]*\n?',
    multiLine: true);

final _examplePattern = RegExp(r'{@example\s+([^}]+)}');

/// Features for processing directives in a documentation comment.
///
/// [processCommentWithoutTools] and [processComment] are the primary
/// entrypoints.
mixin DocumentationComment
    on Documentable, Warnable, Locatable, SourceCodeMixin {
  /// The documentation comment on the Element may be null, so memoization
  /// cannot rely on the null-ness of [_documentationComment], it must be
  /// more explicit.
  bool _documentationCommentComputed = false;
  String _documentationComment;

  String get documentationComment {
    if (_documentationCommentComputed == false) {
      _documentationComment = computeDocumentationComment();
      _documentationCommentComputed = true;
    }
    return _documentationComment;
  }

  /// Implement to derive the raw documentation comment string from the
  /// analyzer.
  String computeDocumentationComment();

  /// Returns true if the raw documentation comment has a nodoc indication.
  bool get hasNodoc {
    if (documentationComment != null &&
        (documentationComment.contains('@nodoc') ||
            documentationComment.contains('<nodoc>'))) {
      return true;
    }
    return packageGraph.configSetsNodocFor(element.source.fullName);
  }

  /// Process a [documentationComment], performing various actions based on
  /// `{@}`-style directives, except `{@tool}`, returning the processed result.
  String processCommentWithoutTools(String documentationComment) {
    var docs = stripComments(documentationComment);
    if (!docs.contains('{@')) {
      _analyzeCodeBlocks(docs);
      return docs;
    }
    docs = _injectExamples(docs);
    docs = _injectYouTube(docs);
    docs = _injectAnimations(docs);

    _analyzeCodeBlocks(docs);

    // TODO(srawlins): Processing templates here causes #2281. But leaving them
    // unprocessed causes #2272.
    docs = _stripHtmlAndAddToIndex(docs);
    return docs;
  }

  /// Process [documentationComment], performing various actions based on
  /// `{@}`-style directives, returning the processed result.
  Future<String> processComment(String documentationComment) async {
    var docs = stripComments(documentationComment);
    // Must evaluate tools first, in case they insert any other directives.
    docs = await _evaluateTools(docs);
    docs = processCommentDirectives(docs);
    _analyzeCodeBlocks(docs);
    return docs;
  }

  String processCommentDirectives(String docs) {
    // The vast, vast majority of doc comments have no directives.
    if (!docs.contains('{@')) {
      return docs;
    }
    _checkForUnknownDirectives(docs);
    docs = _injectExamples(docs);
    docs = _injectYouTube(docs);
    docs = _injectAnimations(docs);
    docs = _stripMacroTemplatesAndAddToIndex(docs);
    docs = _stripHtmlAndAddToIndex(docs);
    return docs;
  }

  String get sourceFileName;

  String get fullyQualifiedNameWithoutLibrary;

  path.Context get pathContext;

  ModelElementRenderer get modelElementRenderer;

  static const _allDirectiveNames = [
    'animation',
    'end-inject-html',
    'end-tool',
    'endtemplate',
    'example',
    'macro',
    'inject-html',
    'template',
    'tool',
    'youtube',

    // Categorization directives, parsed elsewhere:
    'api',
    'canonicalFor',
    'category',
    'image',
    'samples',
    'subCategory',

    // Common Dart annotations which may decorate named parameters:
    'deprecated',
    'required',
  ];

  static final _nameBreak = RegExp('[\\s}]');

  // TODO(srawlins): Implement more checks; see
  // https://github.com/dart-lang/dartdoc/issues/1814.
  void _checkForUnknownDirectives(String docs) {
    var index = 0;
    while (true) {
      var nameStartIndex = docs.indexOf('{@', index);
      if (nameStartIndex == -1) return;
      var nameEndIndex = docs.indexOf(_nameBreak, nameStartIndex + 2);
      if (nameEndIndex == -1) return;
      var name = docs.substring(nameStartIndex + 2, nameEndIndex);
      if (!_allDirectiveNames.contains(name)) {
        if (_allDirectiveNames.contains(name.toLowerCase())) {
          warn(PackageWarning.unknownDirective,
              message: "'$name' (use lowercase)");
        } else {
          warn(PackageWarning.unknownDirective, message: "'$name'");
        }
      }
      // TODO(srawlins): Replace all `replaceAllMapped` usage within this file,
      // running regex after regex over [docs], with simple calls here. This has
      // interactivity / order-of-operations consequences, so take care.
      index = nameEndIndex;
    }
  }

  /// Replace &#123;@tool ...&#125&#123;@end-tool&#125; in API comments with the
  /// output of an external tool.
  ///
  /// Looks for tools invocations, looks up their bound executables in the
  /// options, and executes them with the source comment material as input,
  /// returning the output of the tool. If a named tool isn't configured in the
  /// options file, then it will not be executed, and dartdoc will quit with an
  /// error.
  ///
  /// Tool command line arguments are passed to the tool, with the token
  /// `$INPUT` replaced with the absolute path to a temporary file containing
  /// the content for the tool to read and produce output from. If the tool
  /// doesn't need any input, then no `$INPUT` is needed.
  ///
  /// Nested tool directives will not be evaluated, but tools may generate other
  /// directives in their output and those will be evaluated.
  ///
  /// Syntax:
  ///
  ///     &#123;@tool TOOL [Tool arguments]&#125;
  ///     Content to send to tool.
  ///     &#123;@end-tool&#125;
  ///
  /// Examples:
  ///
  /// In `dart_options.yaml`:
  ///
  /// ```yaml
  /// dartdoc:
  ///   tools:
  ///     prefix:
  ///       # Path is relative to project root.
  ///       command: ["bin/prefix.dart"]
  ///       description: "Prefixes the given input with '##'."
  ///       compile_args: ["--no-sound-null-safety"]
  ///     date:
  ///       command: ["/bin/date"]
  ///       description: "Prints the date"
  /// ```
  ///
  /// In code:
  ///
  /// _This:_
  ///
  ///     &#123;@tool prefix $INPUT&#125;
  ///     Content to send to tool.
  ///     &#123;@end-tool&#125;
  ///     &#123;@tool date --iso-8601=minutes --utc&#125;
  ///     &#123;@end-tool&#125;
  ///
  /// _Produces:_
  ///
  /// ## Content to send to tool.
  /// 2018-09-18T21:15+00:00
  Future<String> _evaluateTools(String rawDocs) async {
    if (!config.allowTools) {
      return rawDocs;
    }
    var invocationIndex = 0;
    return await _replaceAllMappedAsync(rawDocs, _basicToolPattern,
        (basicMatch) async {
      var args = _splitUpQuotedArgs(basicMatch[1]).toList();
      // Tool name must come first.
      if (args.isEmpty) {
        warn(PackageWarning.toolError,
            message: 'Must specify a tool to execute for the @tool directive.');
        return Future.value('');
      }
      // Count the number of invocations of tools in this dartdoc block,
      // so that tools can differentiate different blocks from each other.
      invocationIndex++;
      return await config.tools.runner.run(args, content: basicMatch[2],
          toolErrorCallback: (String message) async {
        print('toolerrocallback for $args');
        warn(PackageWarning.toolError, message: message);
      }, environment: _toolsEnvironment(invocationIndex: invocationIndex));
    });
  }

  /// The environment variables to use when running a tool.
  Map<String, String> _toolsEnvironment({@required int invocationIndex}) {
    return {
      'SOURCE_LINE': characterLocation?.lineNumber.toString(),
      'SOURCE_COLUMN': characterLocation?.columnNumber.toString(),
      if (sourceFileName != null && package?.packagePath != null)
        'SOURCE_PATH':
            pathContext.relative(sourceFileName, from: package.packagePath),
      'PACKAGE_PATH': package?.packagePath,
      'PACKAGE_NAME': package?.name,
      'LIBRARY_NAME': library?.fullyQualifiedName,
      'ELEMENT_NAME': fullyQualifiedNameWithoutLibrary,
      'INVOCATION_INDEX': invocationIndex.toString(),
      'PACKAGE_INVOCATION_INDEX': (package.toolInvocationIndex++).toString(),
    }..removeWhere((key, value) => value == null);
  }

  /// Replace &#123;@example ...&#125; in API comments with the content of named file.
  ///
  /// Syntax:
  ///
  ///     &#123;@example PATH [region=NAME] [lang=NAME]&#125;
  ///
  /// If PATH is `dir/file.ext` and region is `r` then we'll look for the file<commit_msg>Remove stray print (#2690)<commit_after>import 'package:args/args.dart';
import 'package:crypto/crypto.dart' as crypto;
import 'package:dartdoc/src/model/model.dart';
import 'package:dartdoc/src/render/model_element_renderer.dart';
import 'package:dartdoc/src/utils.dart';
import 'package:dartdoc/src/warnings.dart';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path show Context;

final _templatePattern = RegExp(
    r'[ ]*{@template\s+(.+?)}([\s\S]+?){@endtemplate}[ ]*(\n?)',
    multiLine: true);
final _htmlPattern = RegExp(
    r'[ ]*{@inject-html\s*}([\s\S]+?){@end-inject-html}[ ]*\n?',
    multiLine: true);

/// Matches all tool directives (even some invalid ones). This is so
/// we can give good error messages if the directive is malformed, instead of
/// just silently emitting it as-is.
final _basicToolPattern = RegExp(
    r'[ ]*{@tool\s+([^}]+)}\n?([\s\S]+?)\n?{@end-tool}[ ]*\n?',
    multiLine: true);

final _examplePattern = RegExp(r'{@example\s+([^}]+)}');

/// Features for processing directives in a documentation comment.
///
/// [processCommentWithoutTools] and [processComment] are the primary
/// entrypoints.
mixin DocumentationComment
    on Documentable, Warnable, Locatable, SourceCodeMixin {
  /// The documentation comment on the Element may be null, so memoization
  /// cannot rely on the null-ness of [_documentationComment], it must be
  /// more explicit.
  bool _documentationCommentComputed = false;
  String _documentationComment;

  String get documentationComment {
    if (_documentationCommentComputed == false) {
      _documentationComment = computeDocumentationComment();
      _documentationCommentComputed = true;
    }
    return _documentationComment;
  }

  /// Implement to derive the raw documentation comment string from the
  /// analyzer.
  String computeDocumentationComment();

  /// Returns true if the raw documentation comment has a nodoc indication.
  bool get hasNodoc {
    if (documentationComment != null &&
        (documentationComment.contains('@nodoc') ||
            documentationComment.contains('<nodoc>'))) {
      return true;
    }
    return packageGraph.configSetsNodocFor(element.source.fullName);
  }

  /// Process a [documentationComment], performing various actions based on
  /// `{@}`-style directives, except `{@tool}`, returning the processed result.
  String processCommentWithoutTools(String documentationComment) {
    var docs = stripComments(documentationComment);
    if (!docs.contains('{@')) {
      _analyzeCodeBlocks(docs);
      return docs;
    }
    docs = _injectExamples(docs);
    docs = _injectYouTube(docs);
    docs = _injectAnimations(docs);

    _analyzeCodeBlocks(docs);

    // TODO(srawlins): Processing templates here causes #2281. But leaving them
    // unprocessed causes #2272.
    docs = _stripHtmlAndAddToIndex(docs);
    return docs;
  }

  /// Process [documentationComment], performing various actions based on
  /// `{@}`-style directives, returning the processed result.
  Future<String> processComment(String documentationComment) async {
    var docs = stripComments(documentationComment);
    // Must evaluate tools first, in case they insert any other directives.
    docs = await _evaluateTools(docs);
    docs = processCommentDirectives(docs);
    _analyzeCodeBlocks(docs);
    return docs;
  }

  String processCommentDirectives(String docs) {
    // The vast, vast majority of doc comments have no directives.
    if (!docs.contains('{@')) {
      return docs;
    }
    _checkForUnknownDirectives(docs);
    docs = _injectExamples(docs);
    docs = _injectYouTube(docs);
    docs = _injectAnimations(docs);
    docs = _stripMacroTemplatesAndAddToIndex(docs);
    docs = _stripHtmlAndAddToIndex(docs);
    return docs;
  }

  String get sourceFileName;

  String get fullyQualifiedNameWithoutLibrary;

  path.Context get pathContext;

  ModelElementRenderer get modelElementRenderer;

  static const _allDirectiveNames = [
    'animation',
    'end-inject-html',
    'end-tool',
    'endtemplate',
    'example',
    'macro',
    'inject-html',
    'template',
    'tool',
    'youtube',

    // Categorization directives, parsed elsewhere:
    'api',
    'canonicalFor',
    'category',
    'image',
    'samples',
    'subCategory',

    // Common Dart annotations which may decorate named parameters:
    'deprecated',
    'required',
  ];

  static final _nameBreak = RegExp('[\\s}]');

  // TODO(srawlins): Implement more checks; see
  // https://github.com/dart-lang/dartdoc/issues/1814.
  void _checkForUnknownDirectives(String docs) {
    var index = 0;
    while (true) {
      var nameStartIndex = docs.indexOf('{@', index);
      if (nameStartIndex == -1) return;
      var nameEndIndex = docs.indexOf(_nameBreak, nameStartIndex + 2);
      if (nameEndIndex == -1) return;
      var name = docs.substring(nameStartIndex + 2, nameEndIndex);
      if (!_allDirectiveNames.contains(name)) {
        if (_allDirectiveNames.contains(name.toLowerCase())) {
          warn(PackageWarning.unknownDirective,
              message: "'$name' (use lowercase)");
        } else {
          warn(PackageWarning.unknownDirective, message: "'$name'");
        }
      }
      // TODO(srawlins): Replace all `replaceAllMapped` usage within this file,
      // running regex after regex over [docs], with simple calls here. This has
      // interactivity / order-of-operations consequences, so take care.
      index = nameEndIndex;
    }
  }

  /// Replace &#123;@tool ...&#125&#123;@end-tool&#125; in API comments with the
  /// output of an external tool.
  ///
  /// Looks for tools invocations, looks up their bound executables in the
  /// options, and executes them with the source comment material as input,
  /// returning the output of the tool. If a named tool isn't configured in the
  /// options file, then it will not be executed, and dartdoc will quit with an
  /// error.
  ///
  /// Tool command line arguments are passed to the tool, with the token
  /// `$INPUT` replaced with the absolute path to a temporary file containing
  /// the content for the tool to read and produce output from. If the tool
  /// doesn't need any input, then no `$INPUT` is needed.
  ///
  /// Nested tool directives will not be evaluated, but tools may generate other
  /// directives in their output and those will be evaluated.
  ///
  /// Syntax:
  ///
  ///     &#123;@tool TOOL [Tool arguments]&#125;
  ///     Content to send to tool.
  ///     &#123;@end-tool&#125;
  ///
  /// Examples:
  ///
  /// In `dart_options.yaml`:
  ///
  /// ```yaml
  /// dartdoc:
  ///   tools:
  ///     prefix:
  ///       # Path is relative to project root.
  ///       command: ["bin/prefix.dart"]
  ///       description: "Prefixes the given input with '##'."
  ///       compile_args: ["--no-sound-null-safety"]
  ///     date:
  ///       command: ["/bin/date"]
  ///       description: "Prints the date"
  /// ```
  ///
  /// In code:
  ///
  /// _This:_
  ///
  ///     &#123;@tool prefix $INPUT&#125;
  ///     Content to send to tool.
  ///     &#123;@end-tool&#125;
  ///     &#123;@tool date --iso-8601=minutes --utc&#125;
  ///     &#123;@end-tool&#125;
  ///
  /// _Produces:_
  ///
  /// ## Content to send to tool.
  /// 2018-09-18T21:15+00:00
  Future<String> _evaluateTools(String rawDocs) async {
    if (!config.allowTools) {
      return rawDocs;
    }
    var invocationIndex = 0;
    return await _replaceAllMappedAsync(rawDocs, _basicToolPattern,
        (basicMatch) async {
      var args = _splitUpQuotedArgs(basicMatch[1]).toList();
      // Tool name must come first.
      if (args.isEmpty) {
        warn(PackageWarning.toolError,
            message: 'Must specify a tool to execute for the @tool directive.');
        return Future.value('');
      }
      // Count the number of invocations of tools in this dartdoc block,
      // so that tools can differentiate different blocks from each other.
      invocationIndex++;
      return await config.tools.runner.run(args, content: basicMatch[2],
          toolErrorCallback: (String message) async {
        warn(PackageWarning.toolError, message: message);
      }, environment: _toolsEnvironment(invocationIndex: invocationIndex));
    });
  }

  /// The environment variables to use when running a tool.
  Map<String, String> _toolsEnvironment({@required int invocationIndex}) {
    return {
      'SOURCE_LINE': characterLocation?.lineNumber.toString(),
      'SOURCE_COLUMN': characterLocation?.columnNumber.toString(),
      if (sourceFileName != null && package?.packagePath != null)
        'SOURCE_PATH':
            pathContext.relative(sourceFileName, from: package.packagePath),
      'PACKAGE_PATH': package?.packagePath,
      'PACKAGE_NAME': package?.name,
      'LIBRARY_NAME': library?.fullyQualifiedName,
      'ELEMENT_NAME': fullyQualifiedNameWithoutLibrary,
      'INVOCATION_INDEX': invocationIndex.toString(),
      'PACKAGE_INVOCATION_INDEX': (package.toolInvocationIndex++).toString(),
    }..removeWhere((key, value) => value == null);
  }

  /// Replace &#123;@example ...&#125; in API comments with the content of named file.
  ///
  /// Syntax:
  ///
  ///     &#123;@example PATH [region=NAME] [lang=NAME]&#125;
  ///
  /// If PATH is `dir/file.ext` and region is `r` then we'll look for the file
{'fraction_non_alphanumeric': 0.13577148958889482, 'fraction_numerical': 0.010891617725573946, 'mean_word_length': 2.9929652526113837, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6875060', 'n_tokens_mistral': 5729, 'n_tokens_neox': 5409, 'n_words': 1547}