<commit_before>﻿//﻿
// Copyright (c) Microsoft Corporation.  All rights reserved.
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

///<reference path='diagnostics.ts' />
///<reference path='flags.ts' />
///<reference path='nodeTypes.ts' />
///<reference path='hashTable.ts' />
///<reference path='ast.ts' />
///<reference path='astWalker.ts' />
///<reference path='astWalkerCallback.ts' />
///<reference path='astPath.ts' />
///<reference path='binder.ts' />
///<reference path='base64.ts' />
///<reference path='sourceMapping.ts' />
///<reference path='emitter.ts' />
///<reference path='errorReporter.ts' />
///<reference path='printContext.ts' />
///<reference path='scopeAssignment.ts' />
///<reference path='scopeWalk.ts' />
///<reference path='signatures.ts' />
///<reference path='symbols.ts' />
///<reference path='symbolScope.ts' />
///<reference path='typeChecker.ts' />
///<reference path='typeCollection.ts' />
///<reference path='typeFlow.ts' />
///<reference path='types.ts' />
///<reference path='pathUtils.ts' />
///<reference path='referenceResolution.ts' />
///<reference path='precompile.ts' />
///<reference path='declarationEmitter.ts' />
///<reference path='Syntax\ISyntaxNodeOrToken.ts' />
///<reference path='Syntax\Parser.ts' />
///<reference path='Text\TextFactory.ts' />
///<reference path='typecheck\dataMap.ts' />
///<reference path='typecheck\pullFlags.ts' />
///<reference path='typecheck\pullDecls.ts' />
///<reference path='typecheck\pullSymbols.ts' />
///<reference path='typecheck\pullSymbolBindingContext.ts' />
///<reference path='typecheck\pullTypeResolutionContext.ts' />
///<reference path='typecheck\pullTypeResolution.ts' />
///<reference path='typecheck\pullTypeChecker.ts' />
///<reference path='typecheck\pullDeclDiffer.ts' />
///<reference path='typecheck\pullSemanticInfo.ts' />
///<reference path='typecheck\pullDeclCollection.ts' />
///<reference path='typecheck\pullSymbolBinder.ts' />
///<reference path='typecheck\pullSymbolGraph.ts' />
///<reference path='typecheck\pullErrors.ts' />
///<reference path='typecheck\pullHelpers.ts' />
///<reference path='SyntaxTreeToAstVisitor.ts' />
///<reference path='Core\Timer.ts' />

module TypeScript {

    declare var IO;

    export interface EmitterIOHost {
        // function that can even create a folder structure if needed
        createFile(path: string, useUTF8?: boolean): ITextWriter;

        // function to check if file exists on the disk
        fileExists(path: string): boolean;

        // Function to check if the directory exists on the disk
        directoryExists(path: string): boolean;

        // Resolves the path
        resolvePath(path: string): string;
    }

    export interface PullTypeInfoAtPositionInfo {
        symbol: PullSymbol;
        ast: AST;
        enclosingScopeSymbol: PullSymbol;
        candidateSignature: PullSignatureSymbol;
        callSignatures: PullSignatureSymbol[];
        isConstructorCall: boolean;
    }

    export interface PullSymbolInfo {
        symbol: PullSymbol;
        ast: AST;
        enclosingScopeSymbol: PullSymbol;
    }

    export interface PullCallSymbolInfo {
        targetSymbol: PullSymbol;
        resolvedSignatures: TypeScript.PullSignatureSymbol[];
        candidateSignature: TypeScript.PullSignatureSymbol;
        isConstructorCall: boolean;
        ast: AST;
        enclosingScopeSymbol: PullSymbol;
    }

    export interface PullVisibleSymbolsInfo {
        symbols: PullSymbol[];
        enclosingScopeSymbol: PullSymbol;
    }

    export class Document {
        private _diagnostics: IDiagnostic[] = null;
        private _syntaxTree: SyntaxTree = null;
        public script: Script;
        public lineMap: LineMap;

        constructor(public fileName: string,
                    private compilationSettings: CompilationSettings,
                    private scriptSnapshot: IScriptSnapshot,
                    public version: number,
                    public isOpen: boolean,
                    syntaxTree: SyntaxTree) {

            if (isOpen) {
                this._syntaxTree = syntaxTree;
            }
            else {
                // Don't store the syntax tree for a closed file.
                this._diagnostics = syntaxTree.diagnostics();
            }

            this.lineMap = syntaxTree.lineMap();
            this.script = SyntaxTreeToAstVisitor.visit(syntaxTree, fileName, compilationSettings);
        }

        public diagnostics(): IDiagnostic[]{
            if (this._diagnostics === null) {
                this._diagnostics = this._syntaxTree.diagnostics();
            }

            return this._diagnostics;
        }

        public syntaxTree(): SyntaxTree {
            if (this._syntaxTree) {
                return this._syntaxTree;
            }

            return Parser.parse(
                this.fileName,
                SimpleText.fromScriptSnapshot(this.scriptSnapshot),
                TypeScript.isDTSFile(this.fileName),
                LanguageVersion.EcmaScript5);
        }

        public update(scriptSnapshot: IScriptSnapshot, version: number, isOpen: boolean, textChangeRange: TextChangeRange): Document {
            var oldScript = this.script;
            var oldSyntaxTree = this._syntaxTree;

            var text = SimpleText.fromScriptSnapshot(scriptSnapshot);

            // If we don't have a text change, or we don't have an old syntax tree, then do a full
            // parse.  Otherwise, do an incremental parse.
            var newSyntaxTree = textChangeRange === null || oldSyntaxTree === null
                ? TypeScript.Parser.parse(this.fileName, text, TypeScript.isDTSFile(this.fileName))
                : TypeScript.Parser.incrementalParse(oldSyntaxTree, textChangeRange, text);

            return new Document(this.fileName, this.compilationSettings, scriptSnapshot, version, isOpen, newSyntaxTree);
        }

        public static create(fileName: string, scriptSnapshot: IScriptSnapshot, version: number, isOpen: boolean, referencedFiles: IFileReference[], compilationSettings): Document {
            // for an open file, make a syntax tree and a script, and store both around.

            var syntaxTree = Parser.parse(fileName, SimpleText.fromScriptSnapshot(scriptSnapshot), TypeScript.isDTSFile(fileName), LanguageVersion.EcmaScript5);

            var document = new Document(fileName, compilationSettings, scriptSnapshot, version, isOpen, syntaxTree);
            document.script.referencedFiles = referencedFiles;

            return document;
        }

        //public static fromClosed(fileName: string, scriptSnapshot: IScriptSnapshot, script: Script, syntaxTree: SyntaxTree): Document {
        //    return new Document(fileName, scriptSnapshot, script, null, syntaxTree.diagnostics());
        //}
    }

    export class TypeScriptCompiler {
        public pullTypeChecker: PullTypeChecker = null;
        public semanticInfoChain: SemanticInfoChain = null;

        public emitOptions: EmitOptions;

        public fileNameToDocument = new TypeScript.StringHashTable();

        constructor(public logger: ILogger = new NullLogger(),
                    public settings: CompilationSettings = new CompilationSettings(),
                    public diagnosticMessages: IDiagnosticMessages = null) {
            this.emitOptions = new EmitOptions(this.settings);

            if (this.diagnosticMessages) {
                TypeScript.diagnosticMessages = this.diagnosticMessages
            }
        }

        public getDocument(fileName: string): Document {
            return this.fileNameToDocument.lookup(fileName);
        }

        public timeFunction(funcDescription: string, func: () => any): any {
            return TypeScript.timeFunction(this.logger, funcDescription, func);
        }

        public addSourceUnit(fileName: string,
                             scriptSnapshot: IScriptSnapshot,
                             version: number,
                             isOpen: boolean,
                             referencedFiles: IFileReference[] = []): Document {
            return this.timeFunction("addSourceUnit(" + fileName + ")", () => {
                var document = Document.create(fileName, scriptSnapshot, version, isOpen, referencedFiles, this.emitOptions.compilationSettings);
                this.fileNameToDocument.addOrUpdate(fileName, document);

                return document;
            } );
        }

        public updateSourceUnit(fileName: string, scriptSnapshot: IScriptSnapshot, version: number, isOpen: boolean, textChangeRange: TextChangeRange): Document {
            return this.timeFunction("pullUpdateUnit(" + fileName + ")", () => {
                var document = this.getDocument(fileName);
                var updatedDocument = document.update(scriptSnapshot, version, isOpen, textChangeRange);

                this.fileNameToDocument.addOrUpdate(fileName, updatedDocument);

                this.pullUpdateScript(document, updatedDocument);

                return updatedDocument;
            });
        }

        private isDynamicModuleCompilation(): boolean {
            var fileNames = this.fileNameToDocument.getAllKeys();
            for (var i = 0, n = fileNames.length; i < n; i++) {
                var document = this.getDocument(fileNames[i]);
                var script = document.script;
                if (!script.isDeclareFile && script.topLevelMod != null) {
                    return true;
                }
            }
            return false;
        }

        private updateCommonDirectoryPath(): IDiagnostic {
            var commonComponents: string[] = [];
            var commonComponentsLength = -1;

            var fileNames = this.fileNameToDocument.getAllKeys();
            for (var i = 0, len = fileNames.length; i < len; i++) {
                var fileName = fileNames[i];
                var document = this.getDocument(fileNames[i]);
                var script = document.script;

                if (script.emitRequired(this.emitOptions)) {
                    var fileComponents = filePathComponents(fileName);
                    if (commonComponentsLength === -1) {
                        // First time at finding common path
                        // So common path = directory of file
                        commonComponents = fileComponents;
                        commonComponentsLength = commonComponents.length;
                    } else {
                        var updatedPath = false;
                        for (var j = 0; j < commonComponentsLength && j < fileComponents.length; j++) {
                            if (commonComponents[j] != fileComponents[j]) {
                                // The new components = 0 ... j -1
                                commonComponentsLength = j;
                                updatedPath = true;

                                if (j === 0) {
                                    // Its error to not have common path
                                    return new Diagnostic(0, 0, null, "Cannot find the common subdirectory path for the input files");
                                }

                                break;
                            }
                        }

                        // If the fileComponent path completely matched and less than already found update the length
                        if (!updatedPath && fileComponents.length < commonComponentsLength) {
                            commonComponentsLength = fileComponents.length;
                        }
                    }
                }
            }

            this.emitOptions.commonDirectoryPath = commonComponents.slice(0, commonComponentsLength).join("/") + "/";
            if (this.emitOptions.compilationSettings.outputOption.charAt(this.emitOptions.compilationSettings.outputOption.length - 1) != "/") {
                this.emitOptions.compilationSettings.outputOption += "/";
            }

            return null;
        }

        public parseEmitOption(ioHost: EmitterIOHost): IDiagnostic {
            this.emitOptions.ioHost = ioHost;
            if (this.emitOptions.compilationSettings.outputOption === "") {
                this.emitOptions.outputMany = true;
                this.emitOptions.commonDirectoryPath = "";
                return null;
            }

            this.emitOptions.compilationSettings.outputOption = switchToForwardSlashes(this.emitOptions.ioHost.resolvePath(this.emitOptions.compilationSettings.outputOption));

            // Determine if output options is directory or file
            if (this.emitOptions.ioHost.directoryExists(this.emitOptions.compilationSettings.outputOption)) {
                // Existing directory
                this.emitOptions.outputMany = true;
            } else if (this.emitOptions.ioHost.fileExists(this.emitOptions.compilationSettings.outputOption)) {
                // Existing file
                this.emitOptions.outputMany = false;
            }
            else {
                // New File/directory
                this.emitOptions.outputMany = !isJSFile(this.emitOptions.compilationSettings.outputOption);
            }

            // Verify if options are correct
            if (this.isDynamicModuleCompilation() && !this.emitOptions.outputMany) {
                return new Diagnostic(0, 0, null, "Cannot compile dynamic modules when emitting into single file");
            }

            // Parse the directory structure
            if (this.emitOptions.outputMany) {
                return this.updateCommonDirectoryPath();
            }

            return null;
        }

        public getScripts(): Script[] {
            var result: TypeScript.Script[] = [];
            var fileNames = this.fileNameToDocument.getAllKeys();

            for (var i = 0, n = fileNames.length; i < n; i++) {
                var document = this.getDocument(fileNames[i]);
                result.push(document.script);
            }

            return result;
        }

        private useUTF8ForFile(script: Script) {
            if (this.emitOptions.outputMany) {
                return this.outputScriptToUTF8(script);
            } else {
                return this.outputScriptsToUTF8(this.getScripts());
            }
        }

        static mapToDTSFileName(fileName: string, wholeFileNameReplaced: boolean) {
            return getDeclareFilePath(fileName);
        }

        private canEmitDeclarations(script?: Script) {
            if (!this.settings.generateDeclarationFiles) {
                return false;
            }

            // If its already a declare file or is resident or does not contain body 
            if (!!script && (script.isDeclareFile || script.moduleElements === null)) {
                return false;
            }

            return true;
        }

        // Caller is responsible for closing emitter.
        private emitDeclarations(document: Document, declarationEmitter?: DeclarationEmitter): DeclarationEmitter {
            var script = document.script;
            if (this.canEmitDeclarations(script)) {
                if (!declarationEmitter) {
                    var declareFileName = this.emitOptions.mapOutputFileName(document.fileName, TypeScriptCompiler.mapToDTSFileName);
                    declarationEmitter = new DeclarationEmitter(
                        declareFileName, this.useUTF8ForFile(script), this.semanticInfoChain, this.emitOptions);
                }

                declarationEmitter.fileName = document.fileName;
                declarationEmitter.emitDeclarations(script);
            }

            return declarationEmitter;
        }

        // Will not throw exceptions.
        public emitAllDeclarations(): IDiagnostic[] {
            if (this.canEmitDeclarations()) {
                var sharedEmitter: DeclarationEmitter = null;
                var fileNames = this.fileNameToDocument.getAllKeys();

                for (var i = 0, n = fileNames.length; i < n; i++) {
                    var fileName = fileNames[i];

                    try {
                        var document = this.getDocument(fileNames[i]);

                        if (this.emitOptions.outputMany) {
                            var singleEmitter = this.emitDeclarations(document);
                            if (singleEmitter) {
                                singleEmitter.close();
                            }
                        }
                        else {
                            // Create or reuse file
                            sharedEmitter = this.emitDeclarations(document, sharedEmitter);
                        }
                    }
                    catch (ex1) {
                        return Emitter.handleEmitterError(fileName, ex1);
                    }
                }

                if (sharedEmitter) {
                    try {
                        sharedEmitter.close();
                    }
                    catch (ex2) {
                        return Emitter.handleEmitterError(sharedEmitter.fileName, ex2);
                    }
                }
            }

            return [];
        }

        static mapToFileNameExtension(extension: string, fileName: string, wholeFileNameReplaced: boolean) {
            if (wholeFileNameReplaced) {
                // The complete output is redirected in this file so do not change extension
                return fileName;
            } else {
                // Change the extension of the file
                var splitFname = fileName.split(".");
                splitFname.pop();
                return splitFname.join(".") + extension;
            }
        }

        static mapToJSFileName(fileName: string, wholeFileNameReplaced: boolean) {
            return TypeScriptCompiler.mapToFileNameExtension(".js", fileName, wholeFileNameReplaced);
        }

        // Caller is responsible for closing the returned emitter.
        // May throw exceptions.
        private emit(document: Document,
                     inputOutputMapper?: (inputName: string, outputName: string) => void ,
                     emitter?: Emitter): Emitter {

            var script = document.script;
            if (script.emitRequired(this.emitOptions)) {
                var typeScriptFileName = document.fileName;
                if (!emitter) {
                    var javaScriptFileName = this.emitOptions.mapOutputFileName(typeScriptFileName, TypeScriptCompiler.mapToJSFileName);
                    var outFile = this.createFile(javaScriptFileName, this.useUTF8ForFile(script));

                    emitter = new Emitter(javaScriptFileName, outFile, this.emitOptions, this.semanticInfoChain);

                    if (this.settings.mapSourceFiles) {
                        var sourceMapFileName = javaScriptFileName + SourceMapper.MapFileExtension;
                        emitter.setSourceMappings(new SourceMapper(typeScriptFileName, javaScriptFileName, sourceMapFileName, outFile,
                        this.createFile(sourceMapFileName, /*isUTF8:*/ false), this.settings.emitFullSourceMapPath));
                    }

                    if (inputOutputMapper) {
                        // Remember the name of the outfile for this source file
                        inputOutputMapper(typeScriptFileName, javaScriptFileName);
                    }
                }
                else if (this.settings.mapSourceFiles) {
                    emitter.setSourceMappings(new SourceMapper(typeScriptFileName, emitter.emittingFileName, emitter.sourceMapper.sourceMapFileName, emitter.outfile,
                    emitter.sourceMapper.sourceMapOut, this.settings.emitFullSourceMapPath));
                }

                // Set location info
                emitter.setDocument(document);
                emitter.emitJavascript(script, false);
            }

            return emitter;
        }

        // Will not throw exceptions.
        public emitAll(ioHost: EmitterIOHost, inputOutputMapper?: (inputFile: string, outputFile: string) => void ): IDiagnostic[] {
            var optionsDiagnostic = this.parseEmitOption(ioHost);
            if (optionsDiagnostic) {
                return [optionsDiagnostic];
            }
            
            var startEmitTime = (new Date()).getTime();

            var fileNames = this.fileNameToDocument.getAllKeys();
            var sharedEmitter: Emitter = null;

            // Iterate through the files, as long as we don't get an error.
            for (var i = 0, n = fileNames.length; i < n; i++) {
                var fileName = fileNames[i];

                var document = this.getDocument(fileName);

                try {
                    if (this.emitOptions.outputMany) {
                        // We're outputting to mulitple files.  We don't want to reuse an emitter in that case.
                        var singleEmitter = this.emit(document, inputOutputMapper);

                        // Close the emitter after each emitted file.
                        if (singleEmitter) {
                            singleEmitter.emitSourceMapsAndClose();
                        }
                    }
                    else {
                        // We're not outputting to multiple files.  Keep using the same emitter and don't
                        // close until below.
                        sharedEmitter = this.emit(document, inputOutputMapper, sharedEmitter);
                    }
                }
                catch (ex1) {
                    return Emitter.handleEmitterError(fileName, ex1);
                }
            }

            this.logger.log("Emit: " + ((new Date()).getTime() - startEmitTime));

            if (sharedEmitter) {
                try {
                    sharedEmitter.emitSourceMapsAndClose();
                }
                catch (ex2) {
                    return Emitter.handleEmitterError(sharedEmitter.document.fileName, ex2);
                }
            }

            return [];
        }

        private outputScriptToUTF8(script: Script): boolean {
            return script.containsUnicodeChar || (this.emitOptions.compilationSettings.emitComments && script.containsUnicodeCharInComment);
        }

        private outputScriptsToUTF8(scripts: Script[]): boolean {
            for (var i = 0, len = scripts.length; i < len; i++) {
                var script = scripts[i];
                if (this.outputScriptToUTF8(script)) {
                    return true;
                }
            }
            return false;
        }

        private createFile(fileName: string, useUTF8: boolean): ITextWriter {
            // Creating files can cause exceptions, they will be caught higher up in TypeScriptCompiler.emit
            return this.emitOptions.ioHost.createFile(fileName, useUTF8);
        }

        //
        // Pull typecheck infrastructure
        //

        private pullResolveFile(fileName: string): boolean {
            if (!this.pullTypeChecker) {
                return false;
            }

            var unit = this.semanticInfoChain.getUnit(fileName);

            if (!unit) {
                return false;
            }

            this.pullTypeChecker.setUnit(fileName);
            this.pullTypeChecker.resolver.resolveBoundDecls(unit.getTopLevelDecls()[0], new PullTypeResolutionContext());

            return true;
        }

        public getSyntacticDiagnostics(fileName: string): IDiagnostic[]{
            return this.getDocument(fileName).diagnostics();
        }

        /** Used for diagnostics in tests */
        private getSyntaxTree(fileName: string): SyntaxTree {
            return this.getDocument(fileName).syntaxTree();
        }
<commit_msg>Expose getScript method for tests<commit_after>﻿//﻿
// Copyright (c) Microsoft Corporation.  All rights reserved.
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

///<reference path='diagnostics.ts' />
///<reference path='flags.ts' />
///<reference path='nodeTypes.ts' />
///<reference path='hashTable.ts' />
///<reference path='ast.ts' />
///<reference path='astWalker.ts' />
///<reference path='astWalkerCallback.ts' />
///<reference path='astPath.ts' />
///<reference path='binder.ts' />
///<reference path='base64.ts' />
///<reference path='sourceMapping.ts' />
///<reference path='emitter.ts' />
///<reference path='errorReporter.ts' />
///<reference path='printContext.ts' />
///<reference path='scopeAssignment.ts' />
///<reference path='scopeWalk.ts' />
///<reference path='signatures.ts' />
///<reference path='symbols.ts' />
///<reference path='symbolScope.ts' />
///<reference path='typeChecker.ts' />
///<reference path='typeCollection.ts' />
///<reference path='typeFlow.ts' />
///<reference path='types.ts' />
///<reference path='pathUtils.ts' />
///<reference path='referenceResolution.ts' />
///<reference path='precompile.ts' />
///<reference path='declarationEmitter.ts' />
///<reference path='Syntax\ISyntaxNodeOrToken.ts' />
///<reference path='Syntax\Parser.ts' />
///<reference path='Text\TextFactory.ts' />
///<reference path='typecheck\dataMap.ts' />
///<reference path='typecheck\pullFlags.ts' />
///<reference path='typecheck\pullDecls.ts' />
///<reference path='typecheck\pullSymbols.ts' />
///<reference path='typecheck\pullSymbolBindingContext.ts' />
///<reference path='typecheck\pullTypeResolutionContext.ts' />
///<reference path='typecheck\pullTypeResolution.ts' />
///<reference path='typecheck\pullTypeChecker.ts' />
///<reference path='typecheck\pullDeclDiffer.ts' />
///<reference path='typecheck\pullSemanticInfo.ts' />
///<reference path='typecheck\pullDeclCollection.ts' />
///<reference path='typecheck\pullSymbolBinder.ts' />
///<reference path='typecheck\pullSymbolGraph.ts' />
///<reference path='typecheck\pullErrors.ts' />
///<reference path='typecheck\pullHelpers.ts' />
///<reference path='SyntaxTreeToAstVisitor.ts' />
///<reference path='Core\Timer.ts' />

module TypeScript {

    declare var IO;

    export interface EmitterIOHost {
        // function that can even create a folder structure if needed
        createFile(path: string, useUTF8?: boolean): ITextWriter;

        // function to check if file exists on the disk
        fileExists(path: string): boolean;

        // Function to check if the directory exists on the disk
        directoryExists(path: string): boolean;

        // Resolves the path
        resolvePath(path: string): string;
    }

    export interface PullTypeInfoAtPositionInfo {
        symbol: PullSymbol;
        ast: AST;
        enclosingScopeSymbol: PullSymbol;
        candidateSignature: PullSignatureSymbol;
        callSignatures: PullSignatureSymbol[];
        isConstructorCall: boolean;
    }

    export interface PullSymbolInfo {
        symbol: PullSymbol;
        ast: AST;
        enclosingScopeSymbol: PullSymbol;
    }

    export interface PullCallSymbolInfo {
        targetSymbol: PullSymbol;
        resolvedSignatures: TypeScript.PullSignatureSymbol[];
        candidateSignature: TypeScript.PullSignatureSymbol;
        isConstructorCall: boolean;
        ast: AST;
        enclosingScopeSymbol: PullSymbol;
    }

    export interface PullVisibleSymbolsInfo {
        symbols: PullSymbol[];
        enclosingScopeSymbol: PullSymbol;
    }

    export class Document {
        private _diagnostics: IDiagnostic[] = null;
        private _syntaxTree: SyntaxTree = null;
        public script: Script;
        public lineMap: LineMap;

        constructor(public fileName: string,
                    private compilationSettings: CompilationSettings,
                    private scriptSnapshot: IScriptSnapshot,
                    public version: number,
                    public isOpen: boolean,
                    syntaxTree: SyntaxTree) {

            if (isOpen) {
                this._syntaxTree = syntaxTree;
            }
            else {
                // Don't store the syntax tree for a closed file.
                this._diagnostics = syntaxTree.diagnostics();
            }

            this.lineMap = syntaxTree.lineMap();
            this.script = SyntaxTreeToAstVisitor.visit(syntaxTree, fileName, compilationSettings);
        }

        public diagnostics(): IDiagnostic[]{
            if (this._diagnostics === null) {
                this._diagnostics = this._syntaxTree.diagnostics();
            }

            return this._diagnostics;
        }

        public syntaxTree(): SyntaxTree {
            if (this._syntaxTree) {
                return this._syntaxTree;
            }

            return Parser.parse(
                this.fileName,
                SimpleText.fromScriptSnapshot(this.scriptSnapshot),
                TypeScript.isDTSFile(this.fileName),
                LanguageVersion.EcmaScript5);
        }

        public update(scriptSnapshot: IScriptSnapshot, version: number, isOpen: boolean, textChangeRange: TextChangeRange): Document {
            var oldScript = this.script;
            var oldSyntaxTree = this._syntaxTree;

            var text = SimpleText.fromScriptSnapshot(scriptSnapshot);

            // If we don't have a text change, or we don't have an old syntax tree, then do a full
            // parse.  Otherwise, do an incremental parse.
            var newSyntaxTree = textChangeRange === null || oldSyntaxTree === null
                ? TypeScript.Parser.parse(this.fileName, text, TypeScript.isDTSFile(this.fileName))
                : TypeScript.Parser.incrementalParse(oldSyntaxTree, textChangeRange, text);

            return new Document(this.fileName, this.compilationSettings, scriptSnapshot, version, isOpen, newSyntaxTree);
        }

        public static create(fileName: string, scriptSnapshot: IScriptSnapshot, version: number, isOpen: boolean, referencedFiles: IFileReference[], compilationSettings): Document {
            // for an open file, make a syntax tree and a script, and store both around.

            var syntaxTree = Parser.parse(fileName, SimpleText.fromScriptSnapshot(scriptSnapshot), TypeScript.isDTSFile(fileName), LanguageVersion.EcmaScript5);

            var document = new Document(fileName, compilationSettings, scriptSnapshot, version, isOpen, syntaxTree);
            document.script.referencedFiles = referencedFiles;

            return document;
        }

        //public static fromClosed(fileName: string, scriptSnapshot: IScriptSnapshot, script: Script, syntaxTree: SyntaxTree): Document {
        //    return new Document(fileName, scriptSnapshot, script, null, syntaxTree.diagnostics());
        //}
    }

    export class TypeScriptCompiler {
        public pullTypeChecker: PullTypeChecker = null;
        public semanticInfoChain: SemanticInfoChain = null;

        public emitOptions: EmitOptions;

        public fileNameToDocument = new TypeScript.StringHashTable();

        constructor(public logger: ILogger = new NullLogger(),
                    public settings: CompilationSettings = new CompilationSettings(),
                    public diagnosticMessages: IDiagnosticMessages = null) {
            this.emitOptions = new EmitOptions(this.settings);

            if (this.diagnosticMessages) {
                TypeScript.diagnosticMessages = this.diagnosticMessages
            }
        }

        public getDocument(fileName: string): Document {
            return this.fileNameToDocument.lookup(fileName);
        }

        public timeFunction(funcDescription: string, func: () => any): any {
            return TypeScript.timeFunction(this.logger, funcDescription, func);
        }

        public addSourceUnit(fileName: string,
                             scriptSnapshot: IScriptSnapshot,
                             version: number,
                             isOpen: boolean,
                             referencedFiles: IFileReference[] = []): Document {
            return this.timeFunction("addSourceUnit(" + fileName + ")", () => {
                var document = Document.create(fileName, scriptSnapshot, version, isOpen, referencedFiles, this.emitOptions.compilationSettings);
                this.fileNameToDocument.addOrUpdate(fileName, document);

                return document;
            } );
        }

        public updateSourceUnit(fileName: string, scriptSnapshot: IScriptSnapshot, version: number, isOpen: boolean, textChangeRange: TextChangeRange): Document {
            return this.timeFunction("pullUpdateUnit(" + fileName + ")", () => {
                var document = this.getDocument(fileName);
                var updatedDocument = document.update(scriptSnapshot, version, isOpen, textChangeRange);

                this.fileNameToDocument.addOrUpdate(fileName, updatedDocument);

                this.pullUpdateScript(document, updatedDocument);

                return updatedDocument;
            });
        }

        private isDynamicModuleCompilation(): boolean {
            var fileNames = this.fileNameToDocument.getAllKeys();
            for (var i = 0, n = fileNames.length; i < n; i++) {
                var document = this.getDocument(fileNames[i]);
                var script = document.script;
                if (!script.isDeclareFile && script.topLevelMod != null) {
                    return true;
                }
            }
            return false;
        }

        private updateCommonDirectoryPath(): IDiagnostic {
            var commonComponents: string[] = [];
            var commonComponentsLength = -1;

            var fileNames = this.fileNameToDocument.getAllKeys();
            for (var i = 0, len = fileNames.length; i < len; i++) {
                var fileName = fileNames[i];
                var document = this.getDocument(fileNames[i]);
                var script = document.script;

                if (script.emitRequired(this.emitOptions)) {
                    var fileComponents = filePathComponents(fileName);
                    if (commonComponentsLength === -1) {
                        // First time at finding common path
                        // So common path = directory of file
                        commonComponents = fileComponents;
                        commonComponentsLength = commonComponents.length;
                    } else {
                        var updatedPath = false;
                        for (var j = 0; j < commonComponentsLength && j < fileComponents.length; j++) {
                            if (commonComponents[j] != fileComponents[j]) {
                                // The new components = 0 ... j -1
                                commonComponentsLength = j;
                                updatedPath = true;

                                if (j === 0) {
                                    // Its error to not have common path
                                    return new Diagnostic(0, 0, null, "Cannot find the common subdirectory path for the input files");
                                }

                                break;
                            }
                        }

                        // If the fileComponent path completely matched and less than already found update the length
                        if (!updatedPath && fileComponents.length < commonComponentsLength) {
                            commonComponentsLength = fileComponents.length;
                        }
                    }
                }
            }

            this.emitOptions.commonDirectoryPath = commonComponents.slice(0, commonComponentsLength).join("/") + "/";
            if (this.emitOptions.compilationSettings.outputOption.charAt(this.emitOptions.compilationSettings.outputOption.length - 1) != "/") {
                this.emitOptions.compilationSettings.outputOption += "/";
            }

            return null;
        }

        public parseEmitOption(ioHost: EmitterIOHost): IDiagnostic {
            this.emitOptions.ioHost = ioHost;
            if (this.emitOptions.compilationSettings.outputOption === "") {
                this.emitOptions.outputMany = true;
                this.emitOptions.commonDirectoryPath = "";
                return null;
            }

            this.emitOptions.compilationSettings.outputOption = switchToForwardSlashes(this.emitOptions.ioHost.resolvePath(this.emitOptions.compilationSettings.outputOption));

            // Determine if output options is directory or file
            if (this.emitOptions.ioHost.directoryExists(this.emitOptions.compilationSettings.outputOption)) {
                // Existing directory
                this.emitOptions.outputMany = true;
            } else if (this.emitOptions.ioHost.fileExists(this.emitOptions.compilationSettings.outputOption)) {
                // Existing file
                this.emitOptions.outputMany = false;
            }
            else {
                // New File/directory
                this.emitOptions.outputMany = !isJSFile(this.emitOptions.compilationSettings.outputOption);
            }

            // Verify if options are correct
            if (this.isDynamicModuleCompilation() && !this.emitOptions.outputMany) {
                return new Diagnostic(0, 0, null, "Cannot compile dynamic modules when emitting into single file");
            }

            // Parse the directory structure
            if (this.emitOptions.outputMany) {
                return this.updateCommonDirectoryPath();
            }

            return null;
        }

        public getScripts(): Script[] {
            var result: TypeScript.Script[] = [];
            var fileNames = this.fileNameToDocument.getAllKeys();

            for (var i = 0, n = fileNames.length; i < n; i++) {
                var document = this.getDocument(fileNames[i]);
                result.push(document.script);
            }

            return result;
        }

        private useUTF8ForFile(script: Script) {
            if (this.emitOptions.outputMany) {
                return this.outputScriptToUTF8(script);
            } else {
                return this.outputScriptsToUTF8(this.getScripts());
            }
        }

        static mapToDTSFileName(fileName: string, wholeFileNameReplaced: boolean) {
            return getDeclareFilePath(fileName);
        }

        private canEmitDeclarations(script?: Script) {
            if (!this.settings.generateDeclarationFiles) {
                return false;
            }

            // If its already a declare file or is resident or does not contain body 
            if (!!script && (script.isDeclareFile || script.moduleElements === null)) {
                return false;
            }

            return true;
        }

        // Caller is responsible for closing emitter.
        private emitDeclarations(document: Document, declarationEmitter?: DeclarationEmitter): DeclarationEmitter {
            var script = document.script;
            if (this.canEmitDeclarations(script)) {
                if (!declarationEmitter) {
                    var declareFileName = this.emitOptions.mapOutputFileName(document.fileName, TypeScriptCompiler.mapToDTSFileName);
                    declarationEmitter = new DeclarationEmitter(
                        declareFileName, this.useUTF8ForFile(script), this.semanticInfoChain, this.emitOptions);
                }

                declarationEmitter.fileName = document.fileName;
                declarationEmitter.emitDeclarations(script);
            }

            return declarationEmitter;
        }

        // Will not throw exceptions.
        public emitAllDeclarations(): IDiagnostic[] {
            if (this.canEmitDeclarations()) {
                var sharedEmitter: DeclarationEmitter = null;
                var fileNames = this.fileNameToDocument.getAllKeys();

                for (var i = 0, n = fileNames.length; i < n; i++) {
                    var fileName = fileNames[i];

                    try {
                        var document = this.getDocument(fileNames[i]);

                        if (this.emitOptions.outputMany) {
                            var singleEmitter = this.emitDeclarations(document);
                            if (singleEmitter) {
                                singleEmitter.close();
                            }
                        }
                        else {
                            // Create or reuse file
                            sharedEmitter = this.emitDeclarations(document, sharedEmitter);
                        }
                    }
                    catch (ex1) {
                        return Emitter.handleEmitterError(fileName, ex1);
                    }
                }

                if (sharedEmitter) {
                    try {
                        sharedEmitter.close();
                    }
                    catch (ex2) {
                        return Emitter.handleEmitterError(sharedEmitter.fileName, ex2);
                    }
                }
            }

            return [];
        }

        static mapToFileNameExtension(extension: string, fileName: string, wholeFileNameReplaced: boolean) {
            if (wholeFileNameReplaced) {
                // The complete output is redirected in this file so do not change extension
                return fileName;
            } else {
                // Change the extension of the file
                var splitFname = fileName.split(".");
                splitFname.pop();
                return splitFname.join(".") + extension;
            }
        }

        static mapToJSFileName(fileName: string, wholeFileNameReplaced: boolean) {
            return TypeScriptCompiler.mapToFileNameExtension(".js", fileName, wholeFileNameReplaced);
        }

        // Caller is responsible for closing the returned emitter.
        // May throw exceptions.
        private emit(document: Document,
                     inputOutputMapper?: (inputName: string, outputName: string) => void ,
                     emitter?: Emitter): Emitter {

            var script = document.script;
            if (script.emitRequired(this.emitOptions)) {
                var typeScriptFileName = document.fileName;
                if (!emitter) {
                    var javaScriptFileName = this.emitOptions.mapOutputFileName(typeScriptFileName, TypeScriptCompiler.mapToJSFileName);
                    var outFile = this.createFile(javaScriptFileName, this.useUTF8ForFile(script));

                    emitter = new Emitter(javaScriptFileName, outFile, this.emitOptions, this.semanticInfoChain);

                    if (this.settings.mapSourceFiles) {
                        var sourceMapFileName = javaScriptFileName + SourceMapper.MapFileExtension;
                        emitter.setSourceMappings(new SourceMapper(typeScriptFileName, javaScriptFileName, sourceMapFileName, outFile,
                        this.createFile(sourceMapFileName, /*isUTF8:*/ false), this.settings.emitFullSourceMapPath));
                    }

                    if (inputOutputMapper) {
                        // Remember the name of the outfile for this source file
                        inputOutputMapper(typeScriptFileName, javaScriptFileName);
                    }
                }
                else if (this.settings.mapSourceFiles) {
                    emitter.setSourceMappings(new SourceMapper(typeScriptFileName, emitter.emittingFileName, emitter.sourceMapper.sourceMapFileName, emitter.outfile,
                    emitter.sourceMapper.sourceMapOut, this.settings.emitFullSourceMapPath));
                }

                // Set location info
                emitter.setDocument(document);
                emitter.emitJavascript(script, false);
            }

            return emitter;
        }

        // Will not throw exceptions.
        public emitAll(ioHost: EmitterIOHost, inputOutputMapper?: (inputFile: string, outputFile: string) => void ): IDiagnostic[] {
            var optionsDiagnostic = this.parseEmitOption(ioHost);
            if (optionsDiagnostic) {
                return [optionsDiagnostic];
            }
            
            var startEmitTime = (new Date()).getTime();

            var fileNames = this.fileNameToDocument.getAllKeys();
            var sharedEmitter: Emitter = null;

            // Iterate through the files, as long as we don't get an error.
            for (var i = 0, n = fileNames.length; i < n; i++) {
                var fileName = fileNames[i];

                var document = this.getDocument(fileName);

                try {
                    if (this.emitOptions.outputMany) {
                        // We're outputting to mulitple files.  We don't want to reuse an emitter in that case.
                        var singleEmitter = this.emit(document, inputOutputMapper);

                        // Close the emitter after each emitted file.
                        if (singleEmitter) {
                            singleEmitter.emitSourceMapsAndClose();
                        }
                    }
                    else {
                        // We're not outputting to multiple files.  Keep using the same emitter and don't
                        // close until below.
                        sharedEmitter = this.emit(document, inputOutputMapper, sharedEmitter);
                    }
                }
                catch (ex1) {
                    return Emitter.handleEmitterError(fileName, ex1);
                }
            }

            this.logger.log("Emit: " + ((new Date()).getTime() - startEmitTime));

            if (sharedEmitter) {
                try {
                    sharedEmitter.emitSourceMapsAndClose();
                }
                catch (ex2) {
                    return Emitter.handleEmitterError(sharedEmitter.document.fileName, ex2);
                }
            }

            return [];
        }

        private outputScriptToUTF8(script: Script): boolean {
            return script.containsUnicodeChar || (this.emitOptions.compilationSettings.emitComments && script.containsUnicodeCharInComment);
        }

        private outputScriptsToUTF8(scripts: Script[]): boolean {
            for (var i = 0, len = scripts.length; i < len; i++) {
                var script = scripts[i];
                if (this.outputScriptToUTF8(script)) {
                    return true;
                }
            }
            return false;
        }

        private createFile(fileName: string, useUTF8: boolean): ITextWriter {
            // Creating files can cause exceptions, they will be caught higher up in TypeScriptCompiler.emit
            return this.emitOptions.ioHost.createFile(fileName, useUTF8);
        }

        //
        // Pull typecheck infrastructure
        //

        private pullResolveFile(fileName: string): boolean {
            if (!this.pullTypeChecker) {
                return false;
            }

            var unit = this.semanticInfoChain.getUnit(fileName);

            if (!unit) {
                return false;
            }

            this.pullTypeChecker.setUnit(fileName);
            this.pullTypeChecker.resolver.resolveBoundDecls(unit.getTopLevelDecls()[0], new PullTypeResolutionContext());

            return true;
        }

        public getSyntacticDiagnostics(fileName: string): IDiagnostic[]{
            return this.getDocument(fileName).diagnostics();
        }

        /** Used for diagnostics in tests */
        private getSyntaxTree(fileName: string): SyntaxTree {
            return this.getDocument(fileName).syntaxTree();
        }
        private getScript(fileName: string): Script {
            return this.getDocument(fileName).script;
        }

{'fraction_non_alphanumeric': 0.09447638603696099, 'fraction_numerical': 0.0019301848049281315, 'mean_word_length': 1.810051353066759, 'pattern_counts': {'":': 0, '<': 115, '<?xml version=': 0, '>': 107, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5739546', 'n_tokens_mistral': 11798, 'n_tokens_neox': 11323, 'n_words': 2758}