<commit_before>/**
 * fis.baidu.com
 */

var ts = require('typescript');
var util = require('util');

var ModuleKind = ts.ModuleKind;
var JsxEmit = ts.JsxEmit;
var ScriptTarget = ts.ScriptTarget;

function transpileModule(content, transpileOptions, file) {
  var options = transpileOptions.compilerOptions;

  options.isolatedModules = true;
  options.allowNonTsExtensions = true;
  options.noLib = true;
  options.noResolve = true;

  var newLine = ts.getNewLineCharacter(options);
  var inputFileName = file.isJsXLike ? file.realpath.replace(/\.[^\.]+$/, '.tsx') : file.realpath.replace(/\.jsx$/, '.tsx');
  var sourceFile = ts.createSourceFile(inputFileName, content, options.target);
  var sourceMapText, outputText;

  if (file.isMod) {
    sourceFile.moduleName = file.moduleId || file.id;
  }

  var compilerHost = {
    getSourceFile: function (fileName) {
      if (fileName === ts.normalizeSlashes(inputFileName)) {
        return sourceFile;
      }

      var info = fis.uri(fileName, file.dirname);

      if (info.file) {
        var f = info.file;
        var sf = ts.createSourceFile(f.isJsXLike ? f.realpath.replace(/\.[^\.]+$/, '.tsx') : f.realpath.replace(/\.jsx$/, '.tsx'), f.getContent(), options.target);

        if (f.isMod) {
          sf.moduleName = f.moduleId || f.id;
        }

        return sf;
      }

      return undefined;
    },
    writeFile: function (name, text, writeByteOrderMark) {
      if (ts.fileExtensionIs(name, ".map")) {
        ts.Debug.assert(sourceMapText === undefined, "Unexpected multiple source map outputs for the file '" + name + "'");
        sourceMapText = text;
      }
      else {
        ts.Debug.assert(outputText === undefined, "Unexpected multiple outputs for the file: " + name);
        outputText = text;
      }
    },
    getDefaultLibFileName: function () { return "lib.d.ts"; },
    useCaseSensitiveFileNames: function () { return false; },
    getCanonicalFileName: function (fileName) { return fileName; },
    getCurrentDirectory: function () { return ""; },
    getNewLine: function () { return newLine; },
    fileExists: function (fileName) {
      return fis.util.exists(fileName);
    },
    readFile: function (fileName) { return ""; }
  };

  var program = ts.createProgram([inputFileName], options, compilerHost);
  var diagnostics;
  var noticeDiagnostics;

  if (transpileOptions.reportDiagnostics) {
      diagnostics = [];
      ts.addRange(diagnostics, program.getSyntacticDiagnostics(sourceFile));
      ts.addRange(diagnostics, program.getOptionsDiagnostics());

      noticeDiagnostics = [];
      ts.addRange(noticeDiagnostics, program.getSemanticDiagnostics(sourceFile));
  }

  program.emit();

  return {
    outputText: outputText,
    diagnostics: diagnostics,
    noticeDiagnostics: noticeDiagnostics,
    sourceMapText: sourceMapText
  };
}

module.exports = function (content, file, opts) {

  // 用 html 语言处理一遍。
  if (fis.compile.partial && file.ext === '.tsx' || file.ext === '.jsx' || file.isJsXLike) {
    content = fis.compile.partial(content, file, {
      ext: '.html',
      isHtmlLike: true
    });
  }

  var conf = fis.util.clone(opts);

  if (!conf.sourceMap) {
    conf.inlineSources = false;
  }

  var result = transpileModule(content, {
    compilerOptions: conf,
    fileName: file.realpath,
    reportDiagnostics: true,
    moduleName: undefined
  }, file);

  result.diagnostics.forEach(function(e) {
    if(!e.file){
        return;
    }
    let { line, character } = e.file.getLineAndCharacterOfPosition(e.start);
    let message = ts.flattenDiagnosticMessageText(e.messageText, '\n');
    var msg = util.format('Syntax Error: %s in [%s:%s]', message, line+1, character+1);
    throw new Error(msg);
  });
  result.noticeDiagnostics.forEach((e)=>{
    if(!e.file){
        return;
    }
    let { line, character } = e.file.getLineAndCharacterOfPosition(e.start);
    let message = ts.flattenDiagnosticMessageText(e.messageText, '\n');
    var msg = util.format('Notice Syntax Error: %s in [%s:%s]', message, line+1, character+1);
    console.log(msg);
  });

  if (result.sourceMapText) {
    var mapping = fis.file.wrap(file.dirname + '/' + file.filename + file.rExt + '.map');

    // 修改 source 文件名。
    var sourceMapObj = JSON.parse(result.sourceMapText);
    sourceMapObj.sources = [file.subpath];
    result.sourceMapText = JSON.stringify(sourceMapObj, null, 4);

    mapping.setContent(result.sourceMapText);<commit_msg>修改es6语法，兼容node 0.x版本<commit_after>/**
 * fis.baidu.com
 */

var ts = require('typescript');
var util = require('util');

var ModuleKind = ts.ModuleKind;
var JsxEmit = ts.JsxEmit;
var ScriptTarget = ts.ScriptTarget;

function transpileModule(content, transpileOptions, file) {
  var options = transpileOptions.compilerOptions;

  options.isolatedModules = true;
  options.allowNonTsExtensions = true;
  options.noLib = true;
  options.noResolve = true;

  var newLine = ts.getNewLineCharacter(options);
  var inputFileName = file.isJsXLike ? file.realpath.replace(/\.[^\.]+$/, '.tsx') : file.realpath.replace(/\.jsx$/, '.tsx');
  var sourceFile = ts.createSourceFile(inputFileName, content, options.target);
  var sourceMapText, outputText;

  if (file.isMod) {
    sourceFile.moduleName = file.moduleId || file.id;
  }

  var compilerHost = {
    getSourceFile: function (fileName) {
      if (fileName === ts.normalizeSlashes(inputFileName)) {
        return sourceFile;
      }

      var info = fis.uri(fileName, file.dirname);

      if (info.file) {
        var f = info.file;
        var sf = ts.createSourceFile(f.isJsXLike ? f.realpath.replace(/\.[^\.]+$/, '.tsx') : f.realpath.replace(/\.jsx$/, '.tsx'), f.getContent(), options.target);

        if (f.isMod) {
          sf.moduleName = f.moduleId || f.id;
        }

        return sf;
      }

      return undefined;
    },
    writeFile: function (name, text, writeByteOrderMark) {
      if (ts.fileExtensionIs(name, ".map")) {
        ts.Debug.assert(sourceMapText === undefined, "Unexpected multiple source map outputs for the file '" + name + "'");
        sourceMapText = text;
      }
      else {
        ts.Debug.assert(outputText === undefined, "Unexpected multiple outputs for the file: " + name);
        outputText = text;
      }
    },
    getDefaultLibFileName: function () { return "lib.d.ts"; },
    useCaseSensitiveFileNames: function () { return false; },
    getCanonicalFileName: function (fileName) { return fileName; },
    getCurrentDirectory: function () { return ""; },
    getNewLine: function () { return newLine; },
    fileExists: function (fileName) {
      return fis.util.exists(fileName);
    },
    readFile: function (fileName) { return ""; }
  };

  var program = ts.createProgram([inputFileName], options, compilerHost);
  var diagnostics;
  var noticeDiagnostics;

  if (transpileOptions.reportDiagnostics) {
      diagnostics = [];
      ts.addRange(diagnostics, program.getSyntacticDiagnostics(sourceFile));
      ts.addRange(diagnostics, program.getOptionsDiagnostics());

      noticeDiagnostics = [];
      ts.addRange(noticeDiagnostics, program.getSemanticDiagnostics(sourceFile));
  }

  program.emit();

  return {
    outputText: outputText,
    diagnostics: diagnostics,
    noticeDiagnostics: noticeDiagnostics,
    sourceMapText: sourceMapText
  };
}

module.exports = function (content, file, opts) {

  // 用 html 语言处理一遍。
  if (fis.compile.partial && file.ext === '.tsx' || file.ext === '.jsx' || file.isJsXLike) {
    content = fis.compile.partial(content, file, {
      ext: '.html',
      isHtmlLike: true
    });
  }

  var conf = fis.util.clone(opts);

  if (!conf.sourceMap) {
    conf.inlineSources = false;
  }

  var result = transpileModule(content, {
    compilerOptions: conf,
    fileName: file.realpath,
    reportDiagnostics: true,
    moduleName: undefined
  }, file);

  result.diagnostics.forEach(function(e) {
    if(!e.file){
        return;
    }
    var lineAndCharacterObj = e.file.getLineAndCharacterOfPosition(e.start);
    var line = lineAndCharacterObj.line;
    var character = lineAndCharacterObj.character;
    var message = ts.flattenDiagnosticMessageText(e.messageText, '\n');
    var msg = util.format('Syntax Error: %s in [%s:%s]', message, line+1, character+1);
    throw new Error(msg);
  });
  result.noticeDiagnostics.forEach(function(e) {
    if(!e.file){
        return;
    }
    var lineAndCharacterObj = e.file.getLineAndCharacterOfPosition(e.start);
    var line = lineAndCharacterObj.line;
    var character = lineAndCharacterObj.character;
    var message = ts.flattenDiagnosticMessageText(e.messageText, '\n');
    var msg = util.format('Notice Syntax Error: %s in [%s:%s]', message, line+1, character+1);
    console.log(msg);
  });

  if (result.sourceMapText) {
    var mapping = fis.file.wrap(file.dirname + '/' + file.filename + file.rExt + '.map');

    // 修改 source 文件名。
    var sourceMapObj = JSON.parse(result.sourceMapText);
    sourceMapObj.sources = [file.subpath];
    result.sourceMapText = JSON.stringify(sourceMapObj, null, 4);

    mapping.setContent(result.sourceMapText);
{'fraction_non_alphanumeric': 0.14213532513181018, 'fraction_numerical': 0.0013181019332161687, 'mean_word_length': 3.294811320754717, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6897042', 'n_tokens_mistral': 2883, 'n_tokens_neox': 2866, 'n_words': 594}