<commit_before>/// <reference path='../typings/fs-extra/fs-extra.d.ts' />
/// <reference path='../typings/node/node.d.ts' />
/// <reference path='../typings/source-map/source-map.d.ts' />
// Use HEAD version of typescript, installed by npm
/// <reference path='../node_modules/typescript/bin/typescript.d.ts' />
require('source-map-support').install();
import SourceMap = require('source-map');
import fs = require('fs');
import fsx = require('fs-extra');
import path = require('path');
import ts = require('typescript');

import base = require('./base');
import CallTranspiler = require('./call');
import DeclarationTranspiler = require('./declaration');
import ExpressionTranspiler = require('./expression');
import ModuleTranspiler = require('./module');
import StatementTranspiler = require('./statement');
import TypeTranspiler = require('./type');
import LiteralTranspiler = require('./literal');

export interface TranspilerOptions {
  // Fail on the first error, do not collect multiple. Allows easier debugging as stack traces lead
  // directly to the offending line.
  failFast?: boolean;
  // Whether to generate 'library a.b.c;' names from relative file paths.
  generateLibraryName?: boolean;
  // Whether to generate source maps.
  generateSourceMap?: boolean;
  // A base path to relativize absolute file paths against. This is useful for library name
  // generation (see above) and nicer file names in error messages.
  basePath?: string;
}

export class Transpiler {
  private output: Output;
  private currentFile: ts.SourceFile;

  // Comments attach to all following AST nodes before the next 'physical' token. Track the earliest
  // offset to avoid printing comments multiple times.
  private lastCommentIdx: number = -1;
  private errors: string[] = [];

  private transpilers: base.TranspilerStep[];

  constructor(private options: TranspilerOptions = {}) {
    this.transpilers = [
      new CallTranspiler(this),  // Has to come before StatementTranspiler!
      new DeclarationTranspiler(this),
      new ExpressionTranspiler(this),
      new LiteralTranspiler(this),
      new ModuleTranspiler(this, options.generateLibraryName),
      new StatementTranspiler(this),
      new TypeTranspiler(this),
    ];
  }

  private static OPTIONS: ts.CompilerOptions = {
    target: ts.ScriptTarget.ES6,
    module: ts.ModuleKind.CommonJS,
    allowNonTsExtensions: true,
  };

  /**
   * Transpiles the given files to Dart.
   * @param fileNames The input files.
   * @param destination Location to write files to. Creates files next to their sources if absent.
   */
  transpile(fileNames: string[], destination?: string): void {
    var host = this.createCompilerHost(fileNames);
    if (this.options.basePath && destination === undefined) {
      throw new Error('Must have a destination path when a basePath is specified ' +
                      this.options.basePath);
    }
    var destinationRoot = destination || this.options.basePath || '';
    var program = ts.createProgram(fileNames, Transpiler.OPTIONS, host);
    program.getSourceFiles()
        // Do not generate output for .d.ts files.
        .filter((sourceFile: ts.SourceFile) => !sourceFile.fileName.match(/\.d\.ts$/))
        .forEach((f: ts.SourceFile) => {
          var dartCode = this.translate(f);
          var outputFile = this.getOutputPath(f.fileName, destinationRoot);
          fsx.mkdirsSync(path.dirname(outputFile));
          fs.writeFileSync(outputFile, dartCode);
        });
  }

  translateProgram(program: ts.Program): string {
    var src = program.getSourceFiles()
                  .filter((sourceFile: ts.SourceFile) => !sourceFile.fileName.match(/\.d\.ts$/) &&
                                                         !!sourceFile.fileName.match(/\.[jt]s$/))
                  .map((f) => this.translate(f))
                  .join('\n');
    return src;
  }

  translateFile(fileName: string): string {
    var host = this.createCompilerHost([fileName]);
    var program = ts.createProgram([fileName], Transpiler.OPTIONS, host);
    return this.translateProgram(program);
  }

  private createCompilerHost(files: string[]): ts.CompilerHost {
    var fileMap: {[s: string]: boolean} = {};
    files.forEach((f) => fileMap[f] = true);
    return {
      getSourceFile(sourceName, languageVersion) {
        // Only transpile the files directly passed in, do not transpile transitive dependencies.
        if (fileMap.hasOwnProperty(sourceName)) {
          var contents = fs.readFileSync(sourceName, 'UTF-8');
          return ts.createSourceFile(sourceName, contents, Transpiler.OPTIONS.target, true);
        }
        if (sourceName === 'lib.d.ts') {
          return ts.createSourceFile(sourceName, '', Transpiler.OPTIONS.target, true);
        }<commit_msg>Convert OPTIONS from a static field to a local.<commit_after>/// <reference path='../typings/fs-extra/fs-extra.d.ts' />
/// <reference path='../typings/node/node.d.ts' />
/// <reference path='../typings/source-map/source-map.d.ts' />
// Use HEAD version of typescript, installed by npm
/// <reference path='../node_modules/typescript/bin/typescript.d.ts' />
require('source-map-support').install();
import SourceMap = require('source-map');
import fs = require('fs');
import fsx = require('fs-extra');
import path = require('path');
import ts = require('typescript');

import base = require('./base');
import CallTranspiler = require('./call');
import DeclarationTranspiler = require('./declaration');
import ExpressionTranspiler = require('./expression');
import ModuleTranspiler = require('./module');
import StatementTranspiler = require('./statement');
import TypeTranspiler = require('./type');
import LiteralTranspiler = require('./literal');

export interface TranspilerOptions {
  // Fail on the first error, do not collect multiple. Allows easier debugging as stack traces lead
  // directly to the offending line.
  failFast?: boolean;
  // Whether to generate 'library a.b.c;' names from relative file paths.
  generateLibraryName?: boolean;
  // Whether to generate source maps.
  generateSourceMap?: boolean;
  // A base path to relativize absolute file paths against. This is useful for library name
  // generation (see above) and nicer file names in error messages.
  basePath?: string;
}

var OPTIONS: ts.CompilerOptions = {
  target: ts.ScriptTarget.ES6,
  module: ts.ModuleKind.CommonJS,
  allowNonTsExtensions: true,
};

export class Transpiler {
  private output: Output;
  private currentFile: ts.SourceFile;

  // Comments attach to all following AST nodes before the next 'physical' token. Track the earliest
  // offset to avoid printing comments multiple times.
  private lastCommentIdx: number = -1;
  private errors: string[] = [];

  private transpilers: base.TranspilerStep[];

  constructor(private options: TranspilerOptions = {}) {
    this.transpilers = [
      new CallTranspiler(this),  // Has to come before StatementTranspiler!
      new DeclarationTranspiler(this),
      new ExpressionTranspiler(this),
      new LiteralTranspiler(this),
      new ModuleTranspiler(this, options.generateLibraryName),
      new StatementTranspiler(this),
      new TypeTranspiler(this),
    ];
  }

  /**
   * Transpiles the given files to Dart.
   * @param fileNames The input files.
   * @param destination Location to write files to. Creates files next to their sources if absent.
   */
  transpile(fileNames: string[], destination?: string): void {
    var host = this.createCompilerHost(fileNames);
    if (this.options.basePath && destination === undefined) {
      throw new Error('Must have a destination path when a basePath is specified ' +
                      this.options.basePath);
    }
    var destinationRoot = destination || this.options.basePath || '';
    var program = ts.createProgram(fileNames, OPTIONS, host);
    program.getSourceFiles()
        // Do not generate output for .d.ts files.
        .filter((sourceFile: ts.SourceFile) => !sourceFile.fileName.match(/\.d\.ts$/))
        .forEach((f: ts.SourceFile) => {
          var dartCode = this.translate(f);
          var outputFile = this.getOutputPath(f.fileName, destinationRoot);
          fsx.mkdirsSync(path.dirname(outputFile));
          fs.writeFileSync(outputFile, dartCode);
        });
  }

  translateProgram(program: ts.Program): string {
    var src = program.getSourceFiles()
                  .filter((sourceFile: ts.SourceFile) => !sourceFile.fileName.match(/\.d\.ts$/) &&
                                                         !!sourceFile.fileName.match(/\.[jt]s$/))
                  .map((f) => this.translate(f))
                  .join('\n');
    return src;
  }

  translateFile(fileName: string): string {
    var host = this.createCompilerHost([fileName]);
    var program = ts.createProgram([fileName], OPTIONS, host);
    return this.translateProgram(program);
  }

  private createCompilerHost(files: string[]): ts.CompilerHost {
    var fileMap: {[s: string]: boolean} = {};
    files.forEach((f) => fileMap[f] = true);
    return {
      getSourceFile(sourceName, languageVersion) {
        // Only transpile the files directly passed in, do not transpile transitive dependencies.
        if (fileMap.hasOwnProperty(sourceName)) {
          var contents = fs.readFileSync(sourceName, 'UTF-8');
          return ts.createSourceFile(sourceName, contents, OPTIONS.target, true);
        }
        if (sourceName === 'lib.d.ts') {
          return ts.createSourceFile(sourceName, '', OPTIONS.target, true);
        }
{'fraction_non_alphanumeric': 0.12900178440222526, 'fraction_numerical': 0.0006297890206780728, 'mean_word_length': 3.2103402562969507, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7002641', 'n_tokens_mistral': 2656, 'n_tokens_neox': 2606, 'n_words': 727}