<commit_before>#!/usr/bin/env dart
// Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/// Command line tool to merge the SDK libraries and our patch files.
/// This is currently designed as an offline tool, but we could automate it.

import 'dart:io';
import 'dart:isolate' show RawReceivePort;
import 'dart:async';
import 'dart:math' as math;
import 'dart:convert' show JSON;

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/sdk.dart';
import 'package:path/path.dart' as path;

import 'package:front_end/front_end.dart';
import 'package:front_end/src/base/processed_options.dart';
import 'package:front_end/src/kernel_generator_impl.dart';
import 'package:front_end/src/fasta/util/relativize.dart' show relativizeUri;

import 'package:front_end/src/fasta/fasta.dart' as fasta show getDependencies;
import 'package:front_end/src/fasta/kernel/utils.dart' show writeProgramToFile;

import 'package:kernel/target/targets.dart';
import 'package:kernel/target/vm_fasta.dart';
import 'package:kernel/target/flutter_fasta.dart';
import 'package:compiler/src/kernel/dart2js_target.dart' show Dart2jsTarget;

/// Set of input files that were read by this script to generate patched SDK.
/// We will dump it out into the depfile for ninja to use.
///
/// For more information see GN and Ninja references:
///    https://chromium.googlesource.com/chromium/src/+/56807c6cb383140af0c03da8f6731d77785d7160/tools/gn/docs/reference.md#depfile_string_File-name-for-input-dependencies-for-actions
///    https://ninja-build.org/manual.html#_depfile
///
final deps = new Set<Uri>();

/// Create [File] object from the given path and register it as a dependency.
File getInputFile(String path, {canBeMissing: false}) {
  final file = new File(path);
  if (!file.existsSync()) {
    if (!canBeMissing) throw "patch_sdk.dart expects all inputs to exist";
    return null;
  }
  deps.add(Uri.base.resolveUri(file.uri));
  return file;
}

/// Read the given file synchronously as a string and register this path as
/// a dependency.
String readInputFile(String path, {canBeMissing: false}) =>
    getInputFile(path, canBeMissing: canBeMissing)?.readAsStringSync();

Future main(List<String> argv) async {
  var port = new RawReceivePort();
  try {
    await _main(argv);
  } finally {
    port.close();
  }
}

void usage(String mode) {
  var base = path.fromUri(Platform.script);
  final self = path.relative(base);
  print('Usage: $self $mode SDK_DIR PATCH_DIR OUTPUT_DIR PACKAGES');

  final repositoryDir = path.relative(path.dirname(path.dirname(base)));
  final sdkExample = path.relative(path.join(repositoryDir, 'sdk'));
  final patchExample = path.relative(
      path.join(repositoryDir, 'out', 'DebugX64', 'obj', 'gen', 'patch'));
  final outExample = path.relative(
      path.join(repositoryDir, 'out', 'DebugX64', 'obj', 'gen', 'patched_sdk'));
  final packagesExample = path.relative(path.join(repositoryDir, '.packages'));
  print('For example:');
  print('\$ $self vm $sdkExample $patchExample $outExample $packagesExample');

  exit(1);
}

const validModes = const ['vm', 'dart2js', 'flutter'];
String mode;
bool get forVm => mode == 'vm';
bool get forFlutter => mode == 'flutter';
bool get forDart2js => mode == 'dart2js';

Future _main(List<String> argv) async {
  if (argv.isEmpty) usage('[${validModes.join('|')}]');
  mode = argv.first;
  if (!validModes.contains(mode)) usage('[${validModes.join('|')}]');
  if (argv.length != 5) usage(mode);

  var input = argv[1];
  var sdkLibIn = path.join(input, 'lib');
  var patchIn = argv[2];
  var outDir = argv[3];
  var outDirUri = Uri.base.resolveUri(new Uri.directory(outDir));
  var sdkOut = path.join(outDir, 'lib');
  var packagesFile = argv[4];

  // Parse libraries.dart
  var libContents = readInputFile(path.join(
      sdkLibIn, '_internal', 'sdk_library_metadata', 'lib', 'libraries.dart'));
  libContents = _updateLibraryMetadata(sdkOut, libContents);
  var sdkLibraries = _getSdkLibraries(libContents);

  Map<String, String> locations = <String, String>{};

  // Enumerate core libraries and apply patches
  for (SdkLibrary library in sdkLibraries) {
    if (forDart2js && library.isVmLibrary) continue;
    if (!forDart2js && library.isDart2JsLibrary) continue;
    _applyPatch(library, sdkLibIn, patchIn, sdkOut, locations);
  }

  _copyExtraLibraries(sdkOut, locations);

  Uri platform = outDirUri.resolve('platform.dill.tmp');
  Uri outline = outDirUri.resolve('outline.dill');
  Uri librariesJson = outDirUri.resolve("lib/libraries.json");
  Uri packages = Uri.base.resolveUri(new Uri.file(packagesFile));

  await _writeSync(
      librariesJson.toFilePath(), JSON.encode({"libraries": locations}));

  var flags = new TargetFlags();
  var target = forVm
      ? new VmFastaTarget(flags)
      : (forFlutter ? new FlutterFastaTarget(flags) : new Dart2jsTarget(flags));
  var platformDeps =
      await compilePlatform(outDirUri, target, packages, platform, outline);
  deps.addAll(platformDeps);

  if (forVm) {
    // TODO(sigmund): add support for the flutter vmservice_sky as well.
    var vmserviceName = 'vmservice_io';
    var base = path.fromUri(Platform.script);
    Uri dartDir =
        new Uri.directory(path.dirname(path.dirname(path.absolute(base))));
    var program = await kernelForProgram(
        Uri.parse('dart:$vmserviceName'),
        new CompilerOptions()
          ..setExitCodeOnProblem = true
          // TODO(sigmund): investigate. This should be outline, but it breaks
          // vm-debug tests. Issue #30111
          ..sdkSummary = platform
          ..dartLibraries = <String, Uri>{
            '_vmservice': dartDir.resolve('sdk/lib/vmservice/vmservice.dart'),
            'vmservice_io':
                dartDir.resolve('runtime/bin/vmservice/vmservice_io.dart'),
          }
          ..packagesFileUri = packages);
    Uri vmserviceUri = outDirUri.resolve('$vmserviceName.dill');
    // TODO(sigmund): remove. This is a workaround because in the VM
    // doesn't support loading vmservice if it contains external libraries
    // (there is an assertion that only fails in debug builds). Issue #30111
    program.libraries.forEach((l) => l.isExternal = false);
    await writeProgramToFile(program, vmserviceUri);
  }

  Uri platformFinalLocation = outDirUri.resolve('platform.dill');

  // We generate a dependency file for GN to properly regenerate the patched sdk
  // folder, outline.dill and platform.dill files when necessary: either when
  // the sdk sources change or when this script is updated. In particular:
  //
  //  - sdk changes: we track the actual sources we are compiling. If we are
  //    building the dart2js sdk, this includes the dart2js-specific patch
  //    files.
  //
  //    These files are tracked by [deps] and passed below to [writeDepsFile] in
  //    the extraDependencies argument.
  //
  //  - script updates: we track this script file and any code it imports (even
  //    sdk libraries). Note that this script runs on the standalone VM, so any
  //    sdk library used by this script indirectly depends on a VM-specific
  //    patch file.
  //
  //    These set of files is discovered by `getDependencies` below, and the
  //    [platformForDeps] is always the VM-specific `platform.dill` file.
  var platformForDeps = platform;
  var sdkDir = outDirUri;
  if (forDart2js || forFlutter) {
    // Note: this would fail if `../patched_sdk/platform.dill` doesn't exist. We
    // added an explicit dependency in the .GN rules so patched_dart2js_sdk (and
    // patched_flutter_sdk) depend on patched_sdk to ensure that it exists.
    platformForDeps = outDirUri.resolve('../patched_sdk/platform.dill');
    sdkDir = outDirUri.resolve('../patched_sdk/');
  }
  deps.addAll(await fasta.getDependencies(Platform.script,
      sdk: sdkDir, packages: packages, platform: platformForDeps));
  await writeDepsFile(Uri.base.resolveUri(new Uri.file("$outDir.d")),
      platformFinalLocation, deps);
  await new File.fromUri(platform).rename(platformFinalLocation.toFilePath());
}

/// Generates an outline.dill and platform.dill file containing the result of
/// compiling a platform's SDK.
///
/// Returns a list of dependencies read by the compiler. This list can be used
/// to create GN dependency files.
Future<List<Uri>> compilePlatform(Uri patchedSdk, Target target, Uri packages,
    Uri fullOutput, Uri outlineOutput) async {
  var options = new CompilerOptions()
    ..setExitCodeOnProblem = true
    ..strongMode = false
    ..compileSdk = true
    ..sdkRoot = patchedSdk
    ..packagesFileUri = packages
    ..chaseDependencies = true
    ..target = target;

  var result = await generateKernel(
      new ProcessedOptions(
          options,
          // TODO(sigmund): pass all sdk libraries needed here, and make this
          // hermetic.
          false,<commit_msg>Fix bug in writing deps file (Fixes #30156)<commit_after>#!/usr/bin/env dart
// Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/// Command line tool to merge the SDK libraries and our patch files.
/// This is currently designed as an offline tool, but we could automate it.

import 'dart:io';
import 'dart:isolate' show RawReceivePort;
import 'dart:async';
import 'dart:math' as math;
import 'dart:convert' show JSON;

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/sdk.dart';
import 'package:path/path.dart' as path;

import 'package:front_end/front_end.dart';
import 'package:front_end/src/base/processed_options.dart';
import 'package:front_end/src/kernel_generator_impl.dart';
import 'package:front_end/src/fasta/util/relativize.dart' show relativizeUri;

import 'package:front_end/src/fasta/fasta.dart' as fasta show getDependencies;
import 'package:front_end/src/fasta/kernel/utils.dart' show writeProgramToFile;

import 'package:kernel/target/targets.dart';
import 'package:kernel/target/vm_fasta.dart';
import 'package:kernel/target/flutter_fasta.dart';
import 'package:compiler/src/kernel/dart2js_target.dart' show Dart2jsTarget;

/// Set of input files that were read by this script to generate patched SDK.
/// We will dump it out into the depfile for ninja to use.
///
/// For more information see GN and Ninja references:
///    https://chromium.googlesource.com/chromium/src/+/56807c6cb383140af0c03da8f6731d77785d7160/tools/gn/docs/reference.md#depfile_string_File-name-for-input-dependencies-for-actions
///    https://ninja-build.org/manual.html#_depfile
///
final deps = new Set<Uri>();

/// Create [File] object from the given path and register it as a dependency.
File getInputFile(String path, {canBeMissing: false}) {
  final file = new File(path);
  if (!file.existsSync()) {
    if (!canBeMissing) throw "patch_sdk.dart expects all inputs to exist";
    return null;
  }
  deps.add(Uri.base.resolveUri(file.uri));
  return file;
}

/// Read the given file synchronously as a string and register this path as
/// a dependency.
String readInputFile(String path, {canBeMissing: false}) =>
    getInputFile(path, canBeMissing: canBeMissing)?.readAsStringSync();

Future main(List<String> argv) async {
  var port = new RawReceivePort();
  try {
    await _main(argv);
  } finally {
    port.close();
  }
}

void usage(String mode) {
  var base = path.fromUri(Platform.script);
  final self = path.relative(base);
  print('Usage: $self $mode SDK_DIR PATCH_DIR OUTPUT_DIR PACKAGES');

  final repositoryDir = path.relative(path.dirname(path.dirname(base)));
  final sdkExample = path.relative(path.join(repositoryDir, 'sdk'));
  final patchExample = path.relative(
      path.join(repositoryDir, 'out', 'DebugX64', 'obj', 'gen', 'patch'));
  final outExample = path.relative(
      path.join(repositoryDir, 'out', 'DebugX64', 'obj', 'gen', 'patched_sdk'));
  final packagesExample = path.relative(path.join(repositoryDir, '.packages'));
  print('For example:');
  print('\$ $self vm $sdkExample $patchExample $outExample $packagesExample');

  exit(1);
}

const validModes = const ['vm', 'dart2js', 'flutter'];
String mode;
bool get forVm => mode == 'vm';
bool get forFlutter => mode == 'flutter';
bool get forDart2js => mode == 'dart2js';

Future _main(List<String> argv) async {
  if (argv.isEmpty) usage('[${validModes.join('|')}]');
  mode = argv.first;
  if (!validModes.contains(mode)) usage('[${validModes.join('|')}]');
  if (argv.length != 5) usage(mode);

  var input = argv[1];
  var sdkLibIn = path.join(input, 'lib');
  var patchIn = argv[2];
  var outDir = argv[3];
  var outDirUri = Uri.base.resolveUri(new Uri.directory(outDir));
  var sdkOut = path.join(outDir, 'lib');
  var packagesFile = argv[4];

  // Parse libraries.dart
  var libContents = readInputFile(path.join(
      sdkLibIn, '_internal', 'sdk_library_metadata', 'lib', 'libraries.dart'));
  libContents = _updateLibraryMetadata(sdkOut, libContents);
  var sdkLibraries = _getSdkLibraries(libContents);

  Map<String, String> locations = <String, String>{};

  // Enumerate core libraries and apply patches
  for (SdkLibrary library in sdkLibraries) {
    if (forDart2js && library.isVmLibrary) continue;
    if (!forDart2js && library.isDart2JsLibrary) continue;
    _applyPatch(library, sdkLibIn, patchIn, sdkOut, locations);
  }

  _copyExtraLibraries(sdkOut, locations);

  Uri platform = outDirUri.resolve('platform.dill.tmp');
  Uri outline = outDirUri.resolve('outline.dill');
  Uri librariesJson = outDirUri.resolve("lib/libraries.json");
  Uri packages = Uri.base.resolveUri(new Uri.file(packagesFile));

  await _writeSync(
      librariesJson.toFilePath(), JSON.encode({"libraries": locations}));

  var flags = new TargetFlags();
  var target = forVm
      ? new VmFastaTarget(flags)
      : (forFlutter ? new FlutterFastaTarget(flags) : new Dart2jsTarget(flags));
  var platformDeps =
      await compilePlatform(outDirUri, target, packages, platform, outline);
  deps.addAll(platformDeps);

  if (forVm) {
    // TODO(sigmund): add support for the flutter vmservice_sky as well.
    var vmserviceName = 'vmservice_io';
    var base = path.fromUri(Platform.script);
    Uri dartDir =
        new Uri.directory(path.dirname(path.dirname(path.absolute(base))));
    var program = await kernelForProgram(
        Uri.parse('dart:$vmserviceName'),
        new CompilerOptions()
          ..setExitCodeOnProblem = true
          // TODO(sigmund): investigate. This should be outline, but it breaks
          // vm-debug tests. Issue #30111
          ..sdkSummary = platform
          ..dartLibraries = <String, Uri>{
            '_vmservice': dartDir.resolve('sdk/lib/vmservice/vmservice.dart'),
            'vmservice_io':
                dartDir.resolve('runtime/bin/vmservice/vmservice_io.dart'),
          }
          ..packagesFileUri = packages);
    Uri vmserviceUri = outDirUri.resolve('$vmserviceName.dill');
    // TODO(sigmund): remove. This is a workaround because in the VM
    // doesn't support loading vmservice if it contains external libraries
    // (there is an assertion that only fails in debug builds). Issue #30111
    program.libraries.forEach((l) => l.isExternal = false);
    await writeProgramToFile(program, vmserviceUri);
  }

  Uri platformFinalLocation = outDirUri.resolve('platform.dill');

  // We generate a dependency file for GN to properly regenerate the patched sdk
  // folder, outline.dill and platform.dill files when necessary: either when
  // the sdk sources change or when this script is updated. In particular:
  //
  //  - sdk changes: we track the actual sources we are compiling. If we are
  //    building the dart2js sdk, this includes the dart2js-specific patch
  //    files.
  //
  //    These files are tracked by [deps] and passed below to [writeDepsFile] in
  //    the extraDependencies argument.
  //
  //  - script updates: we track this script file and any code it imports (even
  //    sdk libraries). Note that this script runs on the standalone VM, so any
  //    sdk library used by this script indirectly depends on a VM-specific
  //    patch file.
  //
  //    These set of files is discovered by `getDependencies` below, and the
  //    [platformForDeps] is always the VM-specific `platform.dill` file.
  var platformForDeps = platform;
  var sdkDir = outDirUri;
  if (forDart2js || forFlutter) {
    // Note: this would fail if `../patched_sdk/platform.dill` doesn't exist. We
    // added an explicit dependency in the .GN rules so patched_dart2js_sdk (and
    // patched_flutter_sdk) depend on patched_sdk to ensure that it exists.
    platformForDeps = outDirUri.resolve('../patched_sdk/platform.dill');
    sdkDir = outDirUri.resolve('../patched_sdk/');
  }
  deps.addAll(await fasta.getDependencies(Platform.script,
      sdk: sdkDir, packages: packages, platform: platformForDeps));
  await writeDepsFile(platformFinalLocation,
      Uri.base.resolveUri(new Uri.file("$outDir.d")), deps);
  await new File.fromUri(platform).rename(platformFinalLocation.toFilePath());
}

/// Generates an outline.dill and platform.dill file containing the result of
/// compiling a platform's SDK.
///
/// Returns a list of dependencies read by the compiler. This list can be used
/// to create GN dependency files.
Future<List<Uri>> compilePlatform(Uri patchedSdk, Target target, Uri packages,
    Uri fullOutput, Uri outlineOutput) async {
  var options = new CompilerOptions()
    ..setExitCodeOnProblem = true
    ..strongMode = false
    ..compileSdk = true
    ..sdkRoot = patchedSdk
    ..packagesFileUri = packages
    ..chaseDependencies = true
    ..target = target;

  var result = await generateKernel(
      new ProcessedOptions(
          options,
          // TODO(sigmund): pass all sdk libraries needed here, and make this
          // hermetic.
          false,
{'fraction_non_alphanumeric': 0.12090163934426229, 'fraction_numerical': 0.00758750553832521, 'mean_word_length': 3.7694136291600633, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 29, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3178435', 'n_tokens_mistral': 5736, 'n_tokens_neox': 5564, 'n_words': 1612}