<commit_before>import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:yaml/yaml.dart';
import 'package:http/http.dart' as http;
import 'package:safe_config/safe_config.dart';
import 'package:args/args.dart';

Future main(List<String> args) async {
  var parser = new ArgParser()
    ..addFlag("dry-run")
    ..addFlag("docs-only")
    ..addOption("name")
    ..addOption("config", abbr: "c", defaultsTo: "release.yaml");
  var runner = new Runner(parser.parse(args));
  
  try {
    exitCode = await runner.run();
  } catch (e, st) {
    print("Release failed!");
    print("$e");
    print("$st");
    exitCode = -1;
  } finally {
    await runner.cleanup();
  }
}

class Runner {
  Runner(this.options) {
    configuration = new ReleaseConfig(options["config"]);
  }

  ArgResults options;
  ReleaseConfig configuration;
  List<Function> _cleanup = [];
  bool get isDryRun => options["dry-run"];
  bool get docsOnly => options["docs-only"];
  String get name => options["name"];
  Uri baseReferenceURL = Uri.parse("https://www.dartdocs.org/documentation/aqueduct/latest/");

  Future cleanup() async {
    return Future.forEach(_cleanup, (f) => f());
  }

  Future<int> run() async {
    // Ensure we have all the appropriate command line utilities as a pre-check
    // - git
    // - pub
    // - mkdocs

    if (name == null && !(isDryRun || docsOnly)) {
      throw "--name is required.";
    }

    print("Preparing release: '$name'... ${isDryRun ? "(dry-run)":""} ${docsOnly ? "(docs-only)":""}");

    var master = await directoryWithBranch("master");
    var upcomingVersion;
    var changeset;
    if (!docsOnly) {
      var previousVersion = await latestVersion();
      upcomingVersion = await versionFromDirectory(master);
      if (upcomingVersion == previousVersion) {
        throw "Release failed. Version $upcomingVersion already exists.";
      }

      print("Preparing to release $upcomingVersion (from $previousVersion)...");
      changeset = await changesFromDirectory(master, upcomingVersion);
    }

    // Clone docs/source into another directory.
    var docsSource = await directoryWithBranch("docs/source");
    await publishDocs(docsSource, master);

    if (!docsOnly) {
      await postGithubRelease(upcomingVersion, name, changeset);
      await publish(master);
    }

    return 0;
  }

  Future publishDocs(Directory docSource, Directory code) async {
    var symbolMap = await generateSymbolMap(code);
    var blacklist = [
      "tools",
      "build"
    ];
    var transformers = [
      new BlacklistTransformer(blacklist),
      new APIReferenceTransformer(symbolMap, baseReferenceURL)
    ];

    var docsLive = await directoryWithBranch("gh-pages");
    print("Cleaning ${docsLive.path}...");
    docsLive
      .listSync()
      .where((fse) {
        if (fse is Directory) {
          var lastPathComponent = fse.uri.pathSegments[fse.uri.pathSegments.length - 2];
          return lastPathComponent != ".git";
        }
        return true;
      })
      .forEach((fse) {
        if (fse.uri.pathSegments.last != ".nojekyll") {
          fse.deleteSync(recursive: true);
        }
      });

    print("Transforming docs from ${docSource.path} into ${docsLive.path}...");
    await transformDirectory(transformers, docSource, docsLive);

    print("Building /source to /docs site with mkdoc...");
    var process = await Process.start(
        "mkdocs", ["build", "-d", docsLive.uri.resolve("docs").path, "-s"],
        workingDirectory: docsLive.uri.resolve("source").path);
    stderr.addStream(process.stderr);
    stdout.addStream(process.stdout);
    var exitCode = await process.exitCode;
    if (exitCode != 0) {
      throw "mkdocs failed with exit code $exitCode.";
    }<commit_msg>Filter paths consistently<commit_after>import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:yaml/yaml.dart';
import 'package:http/http.dart' as http;
import 'package:safe_config/safe_config.dart';
import 'package:args/args.dart';

Future main(List<String> args) async {
  var parser = new ArgParser()
    ..addFlag("dry-run")
    ..addFlag("docs-only")
    ..addOption("name")
    ..addOption("config", abbr: "c", defaultsTo: "release.yaml");
  var runner = new Runner(parser.parse(args));
  
  try {
    exitCode = await runner.run();
  } catch (e, st) {
    print("Release failed!");
    print("$e");
    print("$st");
    exitCode = -1;
  } finally {
    await runner.cleanup();
  }
}

class Runner {
  Runner(this.options) {
    configuration = new ReleaseConfig(options["config"]);
  }

  ArgResults options;
  ReleaseConfig configuration;
  List<Function> _cleanup = [];
  bool get isDryRun => options["dry-run"];
  bool get docsOnly => options["docs-only"];
  String get name => options["name"];
  Uri baseReferenceURL = Uri.parse("https://www.dartdocs.org/documentation/aqueduct/latest/");

  Future cleanup() async {
    return Future.forEach(_cleanup, (f) => f());
  }

  Future<int> run() async {
    // Ensure we have all the appropriate command line utilities as a pre-check
    // - git
    // - pub
    // - mkdocs

    if (name == null && !(isDryRun || docsOnly)) {
      throw "--name is required.";
    }

    print("Preparing release: '$name'... ${isDryRun ? "(dry-run)":""} ${docsOnly ? "(docs-only)":""}");

    var master = await directoryWithBranch("master");
    var upcomingVersion;
    var changeset;
    if (!docsOnly) {
      var previousVersion = await latestVersion();
      upcomingVersion = await versionFromDirectory(master);
      if (upcomingVersion == previousVersion) {
        throw "Release failed. Version $upcomingVersion already exists.";
      }

      print("Preparing to release $upcomingVersion (from $previousVersion)...");
      changeset = await changesFromDirectory(master, upcomingVersion);
    }

    // Clone docs/source into another directory.
    var docsSource = await directoryWithBranch("docs/source");
    await publishDocs(docsSource, master);

    if (!docsOnly) {
      await postGithubRelease(upcomingVersion, name, changeset);
      await publish(master);
    }

    return 0;
  }

  Future publishDocs(Directory docSource, Directory code) async {
    var symbolMap = await generateSymbolMap(code);
    var blacklist = [
      "tools",
      "build"
    ];
    var transformers = [
      new BlacklistTransformer(blacklist),
      new APIReferenceTransformer(symbolMap, baseReferenceURL)
    ];

    var docsLive = await directoryWithBranch("gh-pages");
    print("Cleaning ${docsLive.path}...");
    docsLive
      .listSync()
      .where((fse) {
        if (fse is Directory) {
          var lastPathComponent = fse.uri.pathSegments[fse.uri.pathSegments.length - 2];
          return lastPathComponent != ".git";
        } else if (fse is File) {
          return fse.uri.pathSegments.last != ".nojekyll";
        }
      })
      .forEach((fse) {
        fse.deleteSync(recursive: true);
      });

    print("Transforming docs from ${docSource.path} into ${docsLive.path}...");
    await transformDirectory(transformers, docSource, docsLive);

    print("Building /source to /docs site with mkdoc...");
    var process = await Process.start(
        "mkdocs", ["build", "-d", docsLive.uri.resolve("docs").path, "-s"],
        workingDirectory: docsLive.uri.resolve("source").path);
    stderr.addStream(process.stderr);
    stdout.addStream(process.stdout);
    var exitCode = await process.exitCode;
    if (exitCode != 0) {
      throw "mkdocs failed with exit code $exitCode.";
    }
{'fraction_non_alphanumeric': 0.14630267691492183, 'fraction_numerical': 0.0010601643254704478, 'mean_word_length': 3.1150490730643403, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5066078', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2253, 'n_words': 549}