<commit_before>// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/error/error.dart';
import 'package:analyzer/error/listener.dart';
import 'package:analyzer/source/line_info.dart';
import 'package:analyzer/src/dart/scanner/reader.dart';
import 'package:analyzer/src/dart/scanner/scanner.dart';
import 'package:analyzer/src/generated/engine.dart';
import 'package:analyzer/src/generated/parser.dart';
import 'package:analyzer/src/test_utilities/resource_provider_mixin.dart';

class ParseBase with ResourceProviderMixin {
  /// Override this to change the analysis options for a given set of tests.
  AnalysisOptionsImpl get analysisOptions => AnalysisOptionsImpl();

  ParseResult parseUnit(String path) {
    var file = getFile(path);
    var source = file.createSource();
    var content = file.readAsStringSync();

    var analysisOptions = this.analysisOptions;
    var experimentStatus = analysisOptions.experimentStatus;

    var errorListener = RecordingErrorListener();

    var reader = CharSequenceReader(content);
    var scanner = Scanner(source, reader, errorListener)
      ..configureFeatures(analysisOptions.contextFeatures);

    scanner.enableGtGtGt = experimentStatus.constant_update_2018;
    var token = scanner.tokenize();

    var useFasta = analysisOptions.useFastaParser;
    var parser = Parser(source, errorListener, useFasta: useFasta);
    parser.enableOptionalNewAndConst = true;
    parser.configureFeatures(experimentStatus);

    var unit = parser.parseCompilationUnit(token);
    unit.lineInfo = LineInfo(scanner.lineStarts);

    return ParseResult(
      path,
      content,
      unit.lineInfo,
      unit,
      errorListener.errors,
    );
  }
}

class ParseResult {
  final String path;<commit_msg>Clean up ParseBase's use of AnalysisOptions.experimentStatus<commit_after>// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/error/error.dart';
import 'package:analyzer/error/listener.dart';
import 'package:analyzer/source/line_info.dart';
import 'package:analyzer/src/dart/scanner/reader.dart';
import 'package:analyzer/src/dart/scanner/scanner.dart';
import 'package:analyzer/src/generated/engine.dart';
import 'package:analyzer/src/generated/parser.dart';
import 'package:analyzer/src/test_utilities/resource_provider_mixin.dart';

class ParseBase with ResourceProviderMixin {
  /// Override this to change the analysis options for a given set of tests.
  AnalysisOptionsImpl get analysisOptions => AnalysisOptionsImpl();

  ParseResult parseUnit(String path) {
    var file = getFile(path);
    var source = file.createSource();
    var content = file.readAsStringSync();

    var analysisOptions = this.analysisOptions;
    var featureSet = analysisOptions.contextFeatures;

    var errorListener = RecordingErrorListener();

    var reader = CharSequenceReader(content);
    var scanner = Scanner(source, reader, errorListener)
      ..configureFeatures(featureSet);

    var token = scanner.tokenize();

    var useFasta = analysisOptions.useFastaParser;
    var parser = Parser(source, errorListener, useFasta: useFasta);
    parser.enableOptionalNewAndConst = true;
    parser.configureFeatures(featureSet);

    var unit = parser.parseCompilationUnit(token);
    unit.lineInfo = LineInfo(scanner.lineStarts);

    return ParseResult(
      path,
      content,
      unit.lineInfo,
      unit,
      errorListener.errors,
    );
  }
}

class ParseResult {
  final String path;
{'fraction_non_alphanumeric': 0.09730423620025674, 'fraction_numerical': 0.0030808729139922978, 'mean_word_length': 4.418636995827538, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2881969', 'n_tokens_mistral': 1138, 'n_tokens_neox': 1102, 'n_words': 282}