<commit_before>// Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

library source_file_provider;

import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:math' as math;
import 'dart:typed_data';

import '../compiler.dart' as api show Diagnostic;
import '../compiler_new.dart' as api;
import '../compiler_new.dart';
import 'colors.dart' as colors;
import 'dart2js.dart' show AbortLeg;
import 'filenames.dart';
import 'io/source_file.dart';
import 'util/uri_extras.dart';

abstract class SourceFileProvider implements CompilerInput {
  bool isWindows = (Platform.operatingSystem == 'windows');
  Uri cwd = currentDirectory;
  Map<Uri, SourceFile> sourceFiles = <Uri, SourceFile>{};
  int dartCharactersRead = 0;

  Future<String> readStringFromUri(Uri resourceUri) {
    return readUtf8BytesFromUri(resourceUri).then(UTF8.decode);
  }

  Future<List<int>> readUtf8BytesFromUri(Uri resourceUri) {
    if (resourceUri.scheme == 'file') {
      return _readFromFile(resourceUri);
    } else if (resourceUri.scheme == 'http' || resourceUri.scheme == 'https') {
      return _readFromHttp(resourceUri);
    } else {
      throw new ArgumentError("Unknown scheme in uri '$resourceUri'");
    }
  }

  Future<List<int>> _readFromFile(Uri resourceUri) {
    assert(resourceUri.scheme == 'file');
    List<int> source;
    try {
      source = _readAll(resourceUri.toFilePath());
    } on FileSystemException catch (ex) {
      String message = ex.osError?.message;
      String detail = message != null ? ' ($message)' : '';
      return new Future.error(
          "Error reading '${relativizeUri(resourceUri)}' $detail");
    }
    dartCharactersRead += source.length;
    sourceFiles[resourceUri] = new CachingUtf8BytesSourceFile(
        resourceUri, relativizeUri(resourceUri), source);
    return new Future.value(source);
  }

  Future<List<int>> _readFromHttp(Uri resourceUri) {
    assert(resourceUri.scheme == 'http');
    HttpClient client = new HttpClient();
    return client
        .getUrl(resourceUri)
        .then((HttpClientRequest request) => request.close())
        .then((HttpClientResponse response) {
      if (response.statusCode != HttpStatus.OK) {
        String msg = 'Failure getting $resourceUri: '
            '${response.statusCode} ${response.reasonPhrase}';
        throw msg;
      }
      return response.toList();
    }).then((List<List<int>> splitContent) {
      int totalLength = splitContent.fold(0, (int old, List list) {
        return old + list.length;
      });
      Uint8List result = new Uint8List(totalLength);
      int offset = 0;
      for (List<int> contentPart in splitContent) {
        result.setRange(offset, offset + contentPart.length, contentPart);
        offset += contentPart.length;
      }
      dartCharactersRead += totalLength;
      sourceFiles[resourceUri] = new CachingUtf8BytesSourceFile(
          resourceUri, resourceUri.toString(), result);
      return result;
    });
  }

  // TODO(johnniwinther): Remove this when no longer needed for the old compiler
  // API.
  Future/*<List<int> | String>*/ call(Uri resourceUri) => throw "unimplemented";

  relativizeUri(Uri uri) => relativize(cwd, uri, isWindows);

  SourceFile getSourceFile(Uri resourceUri) {
    return sourceFiles[resourceUri];
  }
}

List<int> _readAll(String filename) {
  var file = (new File(filename)).openSync();
  var length = file.lengthSync();
  // +1 to have a 0 terminated list, see [Scanner].
  var buffer = new Uint8List(length + 1);
  file.readIntoSync(buffer, 0, length);
  file.closeSync();
  return buffer;
}

class CompilerSourceFileProvider extends SourceFileProvider {
  // TODO(johnniwinther): Remove this when no longer needed for the old compiler
  // API.<commit_msg>dart2js: fix bot failures - I made this private without realizing it was accessed for testing TBR=<EMAIL><commit_after>// Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

library source_file_provider;

import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:math' as math;
import 'dart:typed_data';

import '../compiler.dart' as api show Diagnostic;
import '../compiler_new.dart' as api;
import '../compiler_new.dart';
import 'colors.dart' as colors;
import 'dart2js.dart' show AbortLeg;
import 'filenames.dart';
import 'io/source_file.dart';
import 'util/uri_extras.dart';

abstract class SourceFileProvider implements CompilerInput {
  bool isWindows = (Platform.operatingSystem == 'windows');
  Uri cwd = currentDirectory;
  Map<Uri, SourceFile> sourceFiles = <Uri, SourceFile>{};
  int dartCharactersRead = 0;

  Future<String> readStringFromUri(Uri resourceUri) {
    return readUtf8BytesFromUri(resourceUri).then(UTF8.decode);
  }

  Future<List<int>> readUtf8BytesFromUri(Uri resourceUri) {
    if (resourceUri.scheme == 'file') {
      return _readFromFile(resourceUri);
    } else if (resourceUri.scheme == 'http' || resourceUri.scheme == 'https') {
      return _readFromHttp(resourceUri);
    } else {
      throw new ArgumentError("Unknown scheme in uri '$resourceUri'");
    }
  }

  Future<List<int>> _readFromFile(Uri resourceUri) {
    assert(resourceUri.scheme == 'file');
    List<int> source;
    try {
      source = readAll(resourceUri.toFilePath());
    } on FileSystemException catch (ex) {
      String message = ex.osError?.message;
      String detail = message != null ? ' ($message)' : '';
      return new Future.error(
          "Error reading '${relativizeUri(resourceUri)}' $detail");
    }
    dartCharactersRead += source.length;
    sourceFiles[resourceUri] = new CachingUtf8BytesSourceFile(
        resourceUri, relativizeUri(resourceUri), source);
    return new Future.value(source);
  }

  Future<List<int>> _readFromHttp(Uri resourceUri) {
    assert(resourceUri.scheme == 'http');
    HttpClient client = new HttpClient();
    return client
        .getUrl(resourceUri)
        .then((HttpClientRequest request) => request.close())
        .then((HttpClientResponse response) {
      if (response.statusCode != HttpStatus.OK) {
        String msg = 'Failure getting $resourceUri: '
            '${response.statusCode} ${response.reasonPhrase}';
        throw msg;
      }
      return response.toList();
    }).then((List<List<int>> splitContent) {
      int totalLength = splitContent.fold(0, (int old, List list) {
        return old + list.length;
      });
      Uint8List result = new Uint8List(totalLength);
      int offset = 0;
      for (List<int> contentPart in splitContent) {
        result.setRange(offset, offset + contentPart.length, contentPart);
        offset += contentPart.length;
      }
      dartCharactersRead += totalLength;
      sourceFiles[resourceUri] = new CachingUtf8BytesSourceFile(
          resourceUri, resourceUri.toString(), result);
      return result;
    });
  }

  // TODO(johnniwinther): Remove this when no longer needed for the old compiler
  // API.
  Future/*<List<int> | String>*/ call(Uri resourceUri) => throw "unimplemented";

  relativizeUri(Uri uri) => relativize(cwd, uri, isWindows);

  SourceFile getSourceFile(Uri resourceUri) {
    return sourceFiles[resourceUri];
  }
}

List<int> readAll(String filename) {
  var file = (new File(filename)).openSync();
  var length = file.lengthSync();
  // +1 to have a 0 terminated list, see [Scanner].
  var buffer = new Uint8List(length + 1);
  file.readIntoSync(buffer, 0, length);
  file.closeSync();
  return buffer;
}

class CompilerSourceFileProvider extends SourceFileProvider {
  // TODO(johnniwinther): Remove this when no longer needed for the old compiler
  // API.
{'fraction_non_alphanumeric': 0.12104730584366304, 'fraction_numerical': 0.005185934733114091, 'mean_word_length': 3.6953681710213777, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1781221', 'n_tokens_mistral': 2385, 'n_tokens_neox': 2336, 'n_words': 636}