<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:collection';
import 'dart:convert';
import 'dart:io';

import 'package:path/path.dart' as path;
import 'package:yaml/yaml.dart' as yaml;

import '../base/utils.dart';
import '../cache.dart';
import '../dart/analysis.dart';
import '../dart/sdk.dart';
import '../globals.dart';
import '../runner/flutter_command.dart';

bool isDartFile(FileSystemEntity entry) => entry is File && entry.path.endsWith('.dart');

typedef bool FileFilter(FileSystemEntity entity);

class AnalyzeCommand extends FlutterCommand {
  AnalyzeCommand() {
    argParser.addFlag('flutter-repo', help: 'Include all the examples and tests from the Flutter repository.', defaultsTo: false);
    argParser.addFlag('current-directory', help: 'Include all the Dart files in the current directory, if any.', defaultsTo: true);
    argParser.addFlag('current-package', help: 'Include the lib/main.dart file from the current directory, if any.', defaultsTo: true);
    argParser.addFlag('dartdocs', help: 'List every public member that is lacking documentation (only examines files in the Flutter repository).', defaultsTo: false);
    argParser.addFlag('preamble', help: 'Display the number of files that will be analyzed.', defaultsTo: true);
    argParser.addFlag('congratulate', help: 'Show output even when there are no errors, warnings, hints, or lints.', defaultsTo: true);
    argParser.addFlag('watch', help: 'Run analysis continuously, watching the filesystem for changes.', negatable: false);
    argParser.addOption('dart-sdk', help: 'The path to the Dart SDK.', hide: true);

    // Hidden option to enable a benchmarking mode.
    argParser.addFlag('benchmark', negatable: false, hide: true);

    usesPubOption();
  }

  @override
  String get name => 'analyze';

  @override
  String get description => 'Analyze the project\'s Dart code.';

  @override
  bool get shouldRunPub {
    // If they're not analyzing the current project.
    if (!argResults['current-package'])
      return false;

    // Or we're not in a project directory.
    if (!new File('pubspec.yaml').existsSync())
      return false;

    return super.shouldRunPub;
  }

  @override
  bool get requiresProjectRoot => false;

  @override
  Future<int> runInProject() => argResults['watch'] ? _analyzeWatch() : _analyzeOnce();

  List<String> flutterRootComponents;
  bool isFlutterLibrary(String filename) {
    flutterRootComponents ??= path.normalize(path.absolute(Cache.flutterRoot)).split(path.separator);
    List<String> filenameComponents = path.normalize(path.absolute(filename)).split(path.separator);
    if (filenameComponents.length < flutterRootComponents.length + 4) // the 4: 'packages', package_name, 'lib', file_name
      return false;
    for (int index = 0; index < flutterRootComponents.length; index += 1) {
      if (flutterRootComponents[index] != filenameComponents[index])
        return false;
    }
    if (filenameComponents[flutterRootComponents.length] != 'packages')
      return false;
    if (filenameComponents[flutterRootComponents.length + 1] == 'flutter_tools')
      return false;
    if (filenameComponents[flutterRootComponents.length + 2] != 'lib')
      return false;
    return true;
  }

  bool get _isBenchmarking => argResults['benchmark'];

  Future<int> _analyzeOnce() async {
    Stopwatch stopwatch = new Stopwatch()..start();
    Set<Directory> pubSpecDirectories = new HashSet<Directory>();
    List<File> dartFiles = <File>[];

    for (String file in argResults.rest.toList()) {
      file = path.normalize(path.absolute(file));
      String root = path.rootPrefix(file);
      dartFiles.add(new File(file));
      while (file != root) {
        file = path.dirname(file);
        if (FileSystemEntity.isFileSync(path.join(file, 'pubspec.yaml'))) {
          pubSpecDirectories.add(new Directory(file));
          break;
        }
      }
    }

    bool currentDirectory = argResults['current-directory'] && dartFiles.isEmpty;
    bool currentPackage = argResults['current-package'] && dartFiles.isEmpty;
    bool flutterRepo = argResults['flutter-repo'];

    //TODO (pq): revisit package and directory defaults

    if (currentDirectory  && !flutterRepo) {
      // ./*.dart
      Directory currentDirectory = new Directory('.');
      bool foundOne = false;
      for (FileSystemEntity entry in currentDirectory.listSync()) {
        if (isDartFile(entry)) {<commit_msg>Check for user specified args.<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:collection';
import 'dart:convert';
import 'dart:io';

import 'package:path/path.dart' as path;
import 'package:yaml/yaml.dart' as yaml;

import '../base/utils.dart';
import '../cache.dart';
import '../dart/analysis.dart';
import '../dart/sdk.dart';
import '../globals.dart';
import '../runner/flutter_command.dart';

bool isDartFile(FileSystemEntity entry) => entry is File && entry.path.endsWith('.dart');

typedef bool FileFilter(FileSystemEntity entity);

class AnalyzeCommand extends FlutterCommand {
  AnalyzeCommand() {
    argParser.addFlag('flutter-repo', help: 'Include all the examples and tests from the Flutter repository.', defaultsTo: false);
    argParser.addFlag('current-directory', help: 'Include all the Dart files in the current directory, if any.', defaultsTo: true);
    argParser.addFlag('current-package', help: 'Include the lib/main.dart file from the current directory, if any.', defaultsTo: true);
    argParser.addFlag('dartdocs', help: 'List every public member that is lacking documentation (only examines files in the Flutter repository).', defaultsTo: false);
    argParser.addFlag('preamble', help: 'Display the number of files that will be analyzed.', defaultsTo: true);
    argParser.addFlag('congratulate', help: 'Show output even when there are no errors, warnings, hints, or lints.', defaultsTo: true);
    argParser.addFlag('watch', help: 'Run analysis continuously, watching the filesystem for changes.', negatable: false);
    argParser.addOption('dart-sdk', help: 'The path to the Dart SDK.', hide: true);

    // Hidden option to enable a benchmarking mode.
    argParser.addFlag('benchmark', negatable: false, hide: true);

    usesPubOption();
  }

  @override
  String get name => 'analyze';

  @override
  String get description => 'Analyze the project\'s Dart code.';

  @override
  bool get shouldRunPub {
    // If they're not analyzing the current project.
    if (!argResults['current-package'])
      return false;

    // Or we're not in a project directory.
    if (!new File('pubspec.yaml').existsSync())
      return false;

    return super.shouldRunPub;
  }

  @override
  bool get requiresProjectRoot => false;

  @override
  Future<int> runInProject() => argResults['watch'] ? _analyzeWatch() : _analyzeOnce();

  List<String> flutterRootComponents;
  bool isFlutterLibrary(String filename) {
    flutterRootComponents ??= path.normalize(path.absolute(Cache.flutterRoot)).split(path.separator);
    List<String> filenameComponents = path.normalize(path.absolute(filename)).split(path.separator);
    if (filenameComponents.length < flutterRootComponents.length + 4) // the 4: 'packages', package_name, 'lib', file_name
      return false;
    for (int index = 0; index < flutterRootComponents.length; index += 1) {
      if (flutterRootComponents[index] != filenameComponents[index])
        return false;
    }
    if (filenameComponents[flutterRootComponents.length] != 'packages')
      return false;
    if (filenameComponents[flutterRootComponents.length + 1] == 'flutter_tools')
      return false;
    if (filenameComponents[flutterRootComponents.length + 2] != 'lib')
      return false;
    return true;
  }

  bool get _isBenchmarking => argResults['benchmark'];

  Future<int> _analyzeOnce() async {
    Stopwatch stopwatch = new Stopwatch()..start();
    Set<Directory> pubSpecDirectories = new HashSet<Directory>();
    List<File> dartFiles = <File>[];

    for (String file in argResults.rest.toList()) {
      file = path.normalize(path.absolute(file));
      String root = path.rootPrefix(file);
      dartFiles.add(new File(file));
      while (file != root) {
        file = path.dirname(file);
        if (FileSystemEntity.isFileSync(path.join(file, 'pubspec.yaml'))) {
          pubSpecDirectories.add(new Directory(file));
          break;
        }
      }
    }

    bool currentDirectory = argResults['current-directory'] && (argResults.wasParsed('current-directory') || dartFiles.isEmpty);
    bool currentPackage = argResults['current-package'] && (argResults.wasParsed('current-package') || dartFiles.isEmpty);
    bool flutterRepo = argResults['flutter-repo'];

    //TODO (pq): revisit package and directory defaults

    if (currentDirectory  && !flutterRepo) {
      // ./*.dart
      Directory currentDirectory = new Directory('.');
      bool foundOne = false;
      for (FileSystemEntity entry in currentDirectory.listSync()) {
        if (isDartFile(entry)) {
{'fraction_non_alphanumeric': 0.1268969970939619, 'fraction_numerical': 0.0021526208158432892, 'mean_word_length': 3.9743040685224837, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1365965', 'n_tokens_mistral': 2717, 'n_tokens_neox': 2610, 'n_words': 772}