<commit_before>import { Observable } from "../Misc/observable";
import { Nullable } from "../types";
import { Constants } from "../Engines/constants";
import { DomManagement } from "../Misc/domManagement";
import { Logger } from "../Misc/logger";
import { IDisposable } from '../scene';
import { IPipelineContext } from '../Engines/IPipelineContext';
import { DataBuffer } from '../Meshes/dataBuffer';
import { ShaderProcessor } from '../Engines/Processors/shaderProcessor';
import { IMatrixLike, IVector2Like, IVector3Like, IVector4Like, IColor3Like, IColor4Like } from '../Maths/math.like';
import { ThinEngine } from '../Engines/thinEngine';
import { IEffectFallbacks } from './iEffectFallbacks';
import { WebGLPipelineContext } from '../Engines/WebGL/webGLPipelineContext';

declare type Engine = import("../Engines/engine").Engine;
declare type InternalTexture = import("../Materials/Textures/internalTexture").InternalTexture;
declare type BaseTexture = import("../Materials/Textures/baseTexture").BaseTexture;
declare type RenderTargetTexture = import("../Materials/Textures/renderTargetTexture").RenderTargetTexture;
declare type PostProcess = import("../PostProcesses/postProcess").PostProcess;

/**
 * Options to be used when creating an effect.
 */
export interface IEffectCreationOptions {
    /**
     * Atrributes that will be used in the shader.
     */
    attributes: string[];
    /**
     * Uniform varible names that will be set in the shader.
     */
    uniformsNames: string[];
    /**
     * Uniform buffer variable names that will be set in the shader.
     */
    uniformBuffersNames: string[];
    /**
     * Sampler texture variable names that will be set in the shader.
     */
    samplers: string[];
    /**
     * Define statements that will be set in the shader.
     */
    defines: any;
    /**
     * Possible fallbacks for this effect to improve performance when needed.
     */
    fallbacks: Nullable<IEffectFallbacks>;
    /**
     * Callback that will be called when the shader is compiled.
     */
    onCompiled: Nullable<(effect: Effect) => void>;
    /**
     * Callback that will be called if an error occurs during shader compilation.
     */
    onError: Nullable<(effect: Effect, errors: string) => void>;
    /**
     * Parameters to be used with Babylons include syntax to iterate over an array (eg. {lights: 10})
     */
    indexParameters?: any;
    /**
     * Max number of lights that can be used in the shader.
     */
    maxSimultaneousLights?: number;
    /**
     * See https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext/transformFeedbackVaryings
     */
    transformFeedbackVaryings?: Nullable<string[]>;
}

/**
 * Effect containing vertex and fragment shader that can be executed on an object.
 */
export class Effect implements IDisposable {
    /**
     * Gets or sets the relative url used to load shaders if using the engine in non-minified mode
     */
    public static ShadersRepository = "src/Shaders/";
    /**
     * Enable logging of the shader code when a compilation error occurs
     */
    public static LogShaderCodeOnCompilationError = true;
    /**
     * Name of the effect.
     */
    public name: any = null;
    /**
     * String container all the define statements that should be set on the shader.
     */
    public defines: string = "";
    /**
     * Callback that will be called when the shader is compiled.
     */
    public onCompiled: Nullable<(effect: Effect) => void> = null;
    /**
     * Callback that will be called if an error occurs during shader compilation.
     */
    public onError: Nullable<(effect: Effect, errors: string) => void> = null;
    /**
     * Callback that will be called when effect is bound.
     */
    public onBind: Nullable<(effect: Effect) => void> = null;
    /**
     * Unique ID of the effect.
     */
    public uniqueId = 0;
    /**
     * Observable that will be called when the shader is compiled.
     * It is recommended to use executeWhenCompile() or to make sure that scene.isReady() is called to get this observable raised.
     */
    public onCompileObservable = new Observable<Effect>();
    /**
     * Observable that will be called if an error occurs during shader compilation.
     */
    public onErrorObservable = new Observable<Effect>();

    /** @hidden */
    public _onBindObservable: Nullable<Observable<Effect>> = null;

    /**
     * @hidden
     * Specifies if the effect was previously ready
     */
    public _wasPreviouslyReady = false;

    /**
     * Observable that will be called when effect is bound.
     */
    public get onBindObservable(): Observable<Effect> {
        if (!this._onBindObservable) {
            this._onBindObservable = new Observable<Effect>();
        }

        return this._onBindObservable;
    }

    /** @hidden */
    public _bonesComputationForcedToCPU = false;

    private static _uniqueIdSeed = 0;
    private _engine: Engine;
    private _uniformBuffersNames: { [key: string]: number } = {};
    private _uniformsNames: string[];
    private _samplerList: string[];
    private _samplers: { [key: string]: number } = {};
    private _isReady = false;
    private _compilationError = "";
    private _allFallbacksProcessed = false;
    private _attributesNames: string[];
    private _attributes: number[];
    private _attributeLocationByName: { [name: string] : number };
    private _uniforms: { [key: string]: Nullable<WebGLUniformLocation> } = {};
    /**
     * Key for the effect.
     * @hidden
     */
    public _key: string = "";
    private _indexParameters: any;
    private _fallbacks: Nullable<IEffectFallbacks> = null;
    private _vertexSourceCode: string = "";
    private _fragmentSourceCode: string = "";
    private _vertexSourceCodeOverride: string = "";
    private _fragmentSourceCodeOverride: string = "";
    private _transformFeedbackVaryings: Nullable<string[]> = null;
    /**
     * Compiled shader to webGL program.
     * @hidden
     */
    public _pipelineContext: Nullable<IPipelineContext> = null;
    private _valueCache: { [key: string]: any } = {};
    private static _baseCache: { [key: number]: DataBuffer } = {};

    /**
     * Instantiates an effect.
     * An effect can be used to create/manage/execute vertex and fragment shaders.
     * @param baseName Name of the effect.
     * @param attributesNamesOrOptions List of attribute names that will be passed to the shader or set of all options to create the effect.
     * @param uniformsNamesOrEngine List of uniform variable names that will be passed to the shader or the engine that will be used to render effect.
     * @param samplers List of sampler variables that will be passed to the shader.
     * @param engine Engine to be used to render the effect
     * @param defines Define statements to be added to the shader.
     * @param fallbacks Possible fallbacks for this effect to improve performance when needed.
     * @param onCompiled Callback that will be called when the shader is compiled.
     * @param onError Callback that will be called if an error occurs during shader compilation.
     * @param indexParameters Parameters to be used with Babylons include syntax to iterate over an array (eg. {lights: 10})
     */
    constructor(baseName: any, attributesNamesOrOptions: string[] | IEffectCreationOptions, uniformsNamesOrEngine: string[] | ThinEngine, samplers: Nullable<string[]> = null,
        engine?: ThinEngine, defines: Nullable<string> = null,
        fallbacks: Nullable<IEffectFallbacks> = null, onCompiled: Nullable<(effect: Effect) => void> = null, onError: Nullable<(effect: Effect, errors: string) => void> = null, indexParameters?: any) {
        this.name = baseName;

        if ((<IEffectCreationOptions>attributesNamesOrOptions).attributes) {
            var options = <IEffectCreationOptions>attributesNamesOrOptions;
            this._engine = <Engine>uniformsNamesOrEngine;

            this._attributesNames = options.attributes;
            this._uniformsNames = options.uniformsNames.concat(options.samplers);
            this._samplerList = options.samplers.slice();
            this.defines = options.defines;
            this.onError = options.onError;
            this.onCompiled = options.onCompiled;
            this._fallbacks = options.fallbacks;
            this._indexParameters = options.indexParameters;
            this._transformFeedbackVaryings = options.transformFeedbackVaryings || null;

            if (options.uniformBuffersNames) {
                for (var i = 0; i < options.uniformBuffersNames.length; i++) {
                    this._uniformBuffersNames[options.uniformBuffersNames[i]] = i;
                }
            }
        } else {
            this._engine = <Engine>engine;
            this.defines = (defines == null ? "" : defines);
            this._uniformsNames = (<string[]>uniformsNamesOrEngine).concat(<string[]>samplers);
            this._samplerList = samplers ? <string[]>samplers.slice() : [];
            this._attributesNames = (<string[]>attributesNamesOrOptions);

            this.onError = onError;
            this.onCompiled = onCompiled;

            this._indexParameters = indexParameters;
            this._fallbacks = fallbacks;
        }

        this._attributeLocationByName = { };

        this.uniqueId = Effect._uniqueIdSeed++;

        var vertexSource: any;
        var fragmentSource: any;

        let hostDocument = DomManagement.IsWindowObjectExist() ? this._engine.getHostDocument() : null;

        if (baseName.vertexSource) {
            vertexSource = "source:" + baseName.vertexSource;
        } else if (baseName.vertexElement) {
            vertexSource = hostDocument ? hostDocument.getElementById(baseName.vertexElement) : null;

            if (!vertexSource) {
                vertexSource = baseName.vertexElement;
            }
        } else {
            vertexSource = baseName.vertex || baseName;
        }

        if (baseName.fragmentSource) {
            fragmentSource = "source:" + baseName.fragmentSource;
        } else if (baseName.fragmentElement) {
            fragmentSource = hostDocument ? hostDocument.getElementById(baseName.fragmentElement) : null;

            if (!fragmentSource) {
                fragmentSource = baseName.fragmentElement;
            }
        } else {
            fragmentSource = baseName.fragment || baseName;
        }

        let processorOptions = {
            defines: this.defines.split("\n"),
            indexParameters: this._indexParameters,
            isFragment: false,
            shouldUseHighPrecisionShader: this._engine._shouldUseHighPrecisionShader,
            processor: this._engine._shaderProcessor,
            supportsUniformBuffers: this._engine.supportsUniformBuffers,
            shadersRepository: Effect.ShadersRepository,
            includesShadersStore: Effect.IncludesShadersStore,
            version: (this._engine.webGLVersion * 100).toString(),
            platformName: this._engine.webGLVersion >= 2 ? "WEBGL2" : "WEBGL1"
        };

        this._loadShader(vertexSource, "Vertex", "", (vertexCode) => {
            this._loadShader(fragmentSource, "Fragment", "Pixel", (fragmentCode) => {
                ShaderProcessor.Process(vertexCode, processorOptions, (migratedVertexCode) => {
                    processorOptions.isFragment = true;
                    ShaderProcessor.Process(fragmentCode, processorOptions, (migratedFragmentCode) => {
                        this._useFinalCode(migratedVertexCode, migratedFragmentCode, baseName);
                    });
                });
            });
        });
    }

    private _useFinalCode(migratedVertexCode: string, migratedFragmentCode: string, baseName: any) {
        if (baseName) {
            var vertex = baseName.vertexElement || baseName.vertex || baseName.spectorName || baseName;
            var fragment = baseName.fragmentElement || baseName.fragment || baseName.spectorName || baseName;

            this._vertexSourceCode = "#define SHADER_NAME vertex:" + vertex + "\n" + migratedVertexCode;
            this._fragmentSourceCode = "#define SHADER_NAME fragment:" + fragment + "\n" + migratedFragmentCode;
        } else {
            this._vertexSourceCode = migratedVertexCode;
            this._fragmentSourceCode = migratedFragmentCode;
        }
        this._prepareEffect();
    }

    /**
     * Unique key for this effect
     */
    public get key(): string {
        return this._key;
    }

    /**
     * If the effect has been compiled and prepared.
     * @returns if the effect is compiled and prepared.
     */
    public isReady(): boolean {
        try {
            return this._isReadyInternal();
        }
        catch {
            return false;
        }
    }

    private _isReadyInternal(): boolean {
        if (this._isReady) {
            return true;
        }
        if (this._pipelineContext) {
            return this._pipelineContext.isReady;
        }
        return false;
    }

    /**
     * The engine the effect was initialized with.
     * @returns the engine.
     */
    public getEngine(): Engine {
        return this._engine;
    }

    /**
     * The pipeline context for this effect
     * @returns the associated pipeline context
     */
    public getPipelineContext(): Nullable<IPipelineContext> {
        return this._pipelineContext;
    }

    /**
     * The set of names of attribute variables for the shader.
     * @returns An array of attribute names.
     */
    public getAttributesNames(): string[] {
        return this._attributesNames;
    }

    /**
     * Returns the attribute at the given index.
     * @param index The index of the attribute.
     * @returns The location of the attribute.
     */
    public getAttributeLocation(index: number): number {
        return this._attributes[index];
    }

    /**
     * Returns the attribute based on the name of the variable.
     * @param name of the attribute to look up.
     * @returns the attribute location.
     */
    public getAttributeLocationByName(name: string): number {
        return this._attributeLocationByName[name];
    }

    /**
     * The number of attributes.
     * @returns the numnber of attributes.
     */
    public getAttributesCount(): number {
        return this._attributes.length;
    }

    /**
     * Gets the index of a uniform variable.
     * @param uniformName of the uniform to look up.
     * @returns the index.
     */
    public getUniformIndex(uniformName: string): number {
        return this._uniformsNames.indexOf(uniformName);
    }

    /**
     * Returns the attribute based on the name of the variable.
     * @param uniformName of the uniform to look up.
     * @returns the location of the uniform.
     */
    public getUniform(uniformName: string): Nullable<WebGLUniformLocation> {
        return this._uniforms[uniformName];
    }

    /**
     * Returns an array of sampler variable names
     * @returns The array of sampler variable neames.
     */
    public getSamplers(): string[] {
        return this._samplerList;
    }

    /**
     * The error from the last compilation.
     * @returns the error string.
     */
    public getCompilationError(): string {
        return this._compilationError;
    }

    /**
     * Gets a boolean indicating that all fallbacks were used during compilation
     * @returns true if all fallbacks were used
     */
    public allFallbacksProcessed(): boolean {
        return this._allFallbacksProcessed;
    }

    /**
     * Adds a callback to the onCompiled observable and call the callback imediatly if already ready.
     * @param func The callback to be used.
     */
    public executeWhenCompiled(func: (effect: Effect) => void): void {
        if (this.isReady()) {
            func(this);
            return;
        }

        this.onCompileObservable.add((effect) => {
            func(effect);
        });

        if (!this._pipelineContext || this._pipelineContext.isAsync) {
            setTimeout(() => {
                this._checkIsReady(null);
            }, 16);
        }
    }

    private _checkIsReady(previousPipelineContext: Nullable<IPipelineContext>) {
        try {
            if (this._isReadyInternal()) {
                return;
            }
        } catch (e) {
            this._processCompilationErrors(e, previousPipelineContext);
            return;
        }

        setTimeout(() => {
            this._checkIsReady(previousPipelineContext);
        }, 16);
    }

    private _loadShader(shader: any, key: string, optionalKey: string, callback: (data: any) => void): void {
        if (typeof(HTMLElement) !== "undefined") {
            // DOM element ?
            if (shader instanceof HTMLElement) {
                var shaderCode = DomManagement.GetDOMTextContent(shader);
                callback(shaderCode);
                return;
            }
        }

        // Direct source ?
        if (shader.substr(0, 7) === "source:") {
            callback(shader.substr(7));
            return;
        }

        // Base64 encoded ?
        if (shader.substr(0, 7) === "base64:") {
            var shaderBinary = window.atob(shader.substr(7));
            callback(shaderBinary);
            return;
        }

        // Is in local store ?
        if (Effect.ShadersStore[shader + key + "Shader"]) {
            callback(Effect.ShadersStore[shader + key + "Shader"]);
            return;
        }

        if (optionalKey && Effect.ShadersStore[shader + optionalKey + "Shader"]) {
            callback(Effect.ShadersStore[shader + optionalKey + "Shader"]);
            return;
        }

        var shaderUrl;

        if (shader[0] === "." || shader[0] === "/" || shader.indexOf("http") > -1) {
            shaderUrl = shader;
        } else {
            shaderUrl = Effect.ShadersRepository + shader;
        }

        // Vertex shader
        this._engine._loadFile(shaderUrl + "." + key.toLowerCase() + ".fx", callback);
    }

    /**
     * Recompiles the webGL program
     * @param vertexSourceCode The source code for the vertex shader.
     * @param fragmentSourceCode The source code for the fragment shader.
     * @param onCompiled Callback called when completed.
     * @param onError Callback called on error.
     * @hidden
     */
    public _rebuildProgram(vertexSourceCode: string, fragmentSourceCode: string, onCompiled: (pipelineContext: IPipelineContext) => void, onError: (message: string) => void) {
        this._isReady = false;

        this._vertexSourceCodeOverride = vertexSourceCode;
        this._fragmentSourceCodeOverride = fragmentSourceCode;
        this.onError = (effect, error) => {
            if (onError) {
                onError(error);
            }
        };
        this.onCompiled = () => {
            var scenes = this.getEngine().scenes;
            if (scenes) {
                for (var i = 0; i < scenes.length; i++) {
                    scenes[i].markAllMaterialsAsDirty(Constants.MATERIAL_AllDirtyFlag);
                }
            }

            this._pipelineContext!._handlesSpectorRebuildCallback(onCompiled);
        };
        this._fallbacks = null;
        this._prepareEffect();
    }

    /**
     * Prepares the effect
     * @hidden
     */
    public _prepareEffect() {
        let attributesNames = this._attributesNames;
        let defines = this.defines;
        this._valueCache = {};

        var previousPipelineContext = this._pipelineContext;

        try {
            let engine = this._engine;

            this._pipelineContext = engine.createPipelineContext();

            let rebuildRebind = this._rebuildProgram.bind(this);
            if (this._vertexSourceCodeOverride && this._fragmentSourceCodeOverride) {
                engine._preparePipelineContext(this._pipelineContext, this._vertexSourceCodeOverride, this._fragmentSourceCodeOverride, true, rebuildRebind, null, this._transformFeedbackVaryings);
            }
            else {
                engine._preparePipelineContext(this._pipelineContext, this._vertexSourceCode, this._fragmentSourceCode, false, rebuildRebind, defines, this._transformFeedbackVaryings);
            }

            engine._executeWhenRenderingStateIsCompiled(this._pipelineContext, () => {
                if (engine.supportsUniformBuffers) {
                    for (var name in this._uniformBuffersNames) {
                        this.bindUniformBlock(name, this._uniformBuffersNames[name]);
                    }
                }

                let uniforms = engine.getUniforms(this._pipelineContext!, this._uniformsNames);
                uniforms.forEach((uniform, index) => {
                    this._uniforms[this._uniformsNames[index]] = uniform;
                });

                this._attributes = engine.getAttributes(this._pipelineContext!, attributesNames);
                if (attributesNames) {
                    for (let i = 0; i < attributesNames.length; i++) {
                        const name = attributesNames[i];
                        this._attributeLocationByName[name] = this._attributes[i];
                    }
                }

                var index: number;
                for (index = 0; index < this._samplerList.length; index++) {
                    var sampler = this.getUniform(this._samplerList[index]);

                    if (sampler == null) {
                        this._samplerList.splice(index, 1);
                        index--;
                    }
                }

                this._samplerList.forEach((name, index) => {
                    this._samplers[name] = index;
                });

                engine.bindSamplers(this);

                this._compilationError = "";
                this._isReady = true;
                if (this.onCompiled) {
                    this.onCompiled(this);
                }
                this.onCompileObservable.notifyObservers(this);
                this.onCompileObservable.clear();

                // Unbind mesh reference in fallbacks
                if (this._fallbacks) {
                    this._fallbacks.unBindMesh();
                }

                if (previousPipelineContext) {
                    this.getEngine()._deletePipelineContext(previousPipelineContext);
                }
            });

            if (this._pipelineContext.isAsync) {
                this._checkIsReady(previousPipelineContext);
            }

        } catch (e) {
           this._processCompilationErrors(e, previousPipelineContext);
        }
    }

    private _getShaderCodeAndErrorLine(shader: WebGLShader, error: Nullable<string>, isFragment: boolean): [Nullable<string>, Nullable<string>] {
        const regexp = isFragment ? /FRAGMENT SHADER ERROR: 0:(\d+?):/ : /VERTEX SHADER ERROR: 0:(\d+?):/;
        const code = this._engine._getShaderSource(shader);

        let errorLine = null;

        if (error && code) {
            const res = error.match(regexp);
            if (res && res.length === 2) {
                const lineNumber = parseInt(res[1]);
                const lines = code.split("\n", -1);
                if (lines.length >= lineNumber) {
                    errorLine = `Offending line [${lineNumber}] in ${isFragment ? "fragment" : "vertex"} code: ${lines[lineNumber - 1]}`;
                }
            }
        }

        return [code, errorLine];
    }

    private _processCompilationErrors(e: any, previousPipelineContext: Nullable<IPipelineContext> = null) {
        this._compilationError = e.message;
        let attributesNames = this._attributesNames;
        let fallbacks = this._fallbacks;

        // Let's go through fallbacks then
        Logger.Error("Unable to compile effect:");
        Logger.Error("Uniforms: " + this._uniformsNames.map(function(uniform) {
            return " " + uniform;
        }));
        Logger.Error("Attributes: " + attributesNames.map(function(attribute) {
            return " " + attribute;
        }));
        Logger.Error("Defines:\r\n" + this.defines);
        if (Effect.LogShaderCodeOnCompilationError) {
            let pcw = this._pipelineContext as WebGLPipelineContext;
            let lineErrorVertex = null, lineErrorFragment = null, code = null;
            if (pcw.vertexShader) {
                [code, lineErrorVertex] = this._getShaderCodeAndErrorLine(pcw.vertexShader, this._compilationError, false);
                if (code) {
                    Logger.Error("Vertex code:");
                    Logger.Error(code);
                }
            }
            if (pcw.fragmentShader) {
                [code, lineErrorFragment] = this._getShaderCodeAndErrorLine(pcw.fragmentShader, this._compilationError, true);
                if (code) {
                    Logger.Error("Fragment code:");
                    Logger.Error(code);
                }
            }
            if (lineErrorVertex) {
                Logger.Error(lineErrorVertex);
            }
            if (lineErrorFragment) {
                Logger.Error(lineErrorFragment);
            }
        }
        Logger.Error("Error: " + this._compilationError);<commit_msg>Add type guard around WebGLPipelineContext<commit_after>import { Observable } from "../Misc/observable";
import { Nullable } from "../types";
import { Constants } from "../Engines/constants";
import { DomManagement } from "../Misc/domManagement";
import { Logger } from "../Misc/logger";
import { IDisposable } from '../scene';
import { IPipelineContext } from '../Engines/IPipelineContext';
import { DataBuffer } from '../Meshes/dataBuffer';
import { ShaderProcessor } from '../Engines/Processors/shaderProcessor';
import { IMatrixLike, IVector2Like, IVector3Like, IVector4Like, IColor3Like, IColor4Like } from '../Maths/math.like';
import { ThinEngine } from '../Engines/thinEngine';
import { IEffectFallbacks } from './iEffectFallbacks';
import { WebGLPipelineContext } from '../Engines/WebGL/webGLPipelineContext';

declare type Engine = import("../Engines/engine").Engine;
declare type InternalTexture = import("../Materials/Textures/internalTexture").InternalTexture;
declare type BaseTexture = import("../Materials/Textures/baseTexture").BaseTexture;
declare type RenderTargetTexture = import("../Materials/Textures/renderTargetTexture").RenderTargetTexture;
declare type PostProcess = import("../PostProcesses/postProcess").PostProcess;

/**
 * Options to be used when creating an effect.
 */
export interface IEffectCreationOptions {
    /**
     * Atrributes that will be used in the shader.
     */
    attributes: string[];
    /**
     * Uniform varible names that will be set in the shader.
     */
    uniformsNames: string[];
    /**
     * Uniform buffer variable names that will be set in the shader.
     */
    uniformBuffersNames: string[];
    /**
     * Sampler texture variable names that will be set in the shader.
     */
    samplers: string[];
    /**
     * Define statements that will be set in the shader.
     */
    defines: any;
    /**
     * Possible fallbacks for this effect to improve performance when needed.
     */
    fallbacks: Nullable<IEffectFallbacks>;
    /**
     * Callback that will be called when the shader is compiled.
     */
    onCompiled: Nullable<(effect: Effect) => void>;
    /**
     * Callback that will be called if an error occurs during shader compilation.
     */
    onError: Nullable<(effect: Effect, errors: string) => void>;
    /**
     * Parameters to be used with Babylons include syntax to iterate over an array (eg. {lights: 10})
     */
    indexParameters?: any;
    /**
     * Max number of lights that can be used in the shader.
     */
    maxSimultaneousLights?: number;
    /**
     * See https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext/transformFeedbackVaryings
     */
    transformFeedbackVaryings?: Nullable<string[]>;
}

/**
 * Effect containing vertex and fragment shader that can be executed on an object.
 */
export class Effect implements IDisposable {
    /**
     * Gets or sets the relative url used to load shaders if using the engine in non-minified mode
     */
    public static ShadersRepository = "src/Shaders/";
    /**
     * Enable logging of the shader code when a compilation error occurs
     */
    public static LogShaderCodeOnCompilationError = true;
    /**
     * Name of the effect.
     */
    public name: any = null;
    /**
     * String container all the define statements that should be set on the shader.
     */
    public defines: string = "";
    /**
     * Callback that will be called when the shader is compiled.
     */
    public onCompiled: Nullable<(effect: Effect) => void> = null;
    /**
     * Callback that will be called if an error occurs during shader compilation.
     */
    public onError: Nullable<(effect: Effect, errors: string) => void> = null;
    /**
     * Callback that will be called when effect is bound.
     */
    public onBind: Nullable<(effect: Effect) => void> = null;
    /**
     * Unique ID of the effect.
     */
    public uniqueId = 0;
    /**
     * Observable that will be called when the shader is compiled.
     * It is recommended to use executeWhenCompile() or to make sure that scene.isReady() is called to get this observable raised.
     */
    public onCompileObservable = new Observable<Effect>();
    /**
     * Observable that will be called if an error occurs during shader compilation.
     */
    public onErrorObservable = new Observable<Effect>();

    /** @hidden */
    public _onBindObservable: Nullable<Observable<Effect>> = null;

    /**
     * @hidden
     * Specifies if the effect was previously ready
     */
    public _wasPreviouslyReady = false;

    /**
     * Observable that will be called when effect is bound.
     */
    public get onBindObservable(): Observable<Effect> {
        if (!this._onBindObservable) {
            this._onBindObservable = new Observable<Effect>();
        }

        return this._onBindObservable;
    }

    /** @hidden */
    public _bonesComputationForcedToCPU = false;

    private static _uniqueIdSeed = 0;
    private _engine: Engine;
    private _uniformBuffersNames: { [key: string]: number } = {};
    private _uniformsNames: string[];
    private _samplerList: string[];
    private _samplers: { [key: string]: number } = {};
    private _isReady = false;
    private _compilationError = "";
    private _allFallbacksProcessed = false;
    private _attributesNames: string[];
    private _attributes: number[];
    private _attributeLocationByName: { [name: string] : number };
    private _uniforms: { [key: string]: Nullable<WebGLUniformLocation> } = {};
    /**
     * Key for the effect.
     * @hidden
     */
    public _key: string = "";
    private _indexParameters: any;
    private _fallbacks: Nullable<IEffectFallbacks> = null;
    private _vertexSourceCode: string = "";
    private _fragmentSourceCode: string = "";
    private _vertexSourceCodeOverride: string = "";
    private _fragmentSourceCodeOverride: string = "";
    private _transformFeedbackVaryings: Nullable<string[]> = null;
    /**
     * Compiled shader to webGL program.
     * @hidden
     */
    public _pipelineContext: Nullable<IPipelineContext> = null;
    private _valueCache: { [key: string]: any } = {};
    private static _baseCache: { [key: number]: DataBuffer } = {};

    /**
     * Instantiates an effect.
     * An effect can be used to create/manage/execute vertex and fragment shaders.
     * @param baseName Name of the effect.
     * @param attributesNamesOrOptions List of attribute names that will be passed to the shader or set of all options to create the effect.
     * @param uniformsNamesOrEngine List of uniform variable names that will be passed to the shader or the engine that will be used to render effect.
     * @param samplers List of sampler variables that will be passed to the shader.
     * @param engine Engine to be used to render the effect
     * @param defines Define statements to be added to the shader.
     * @param fallbacks Possible fallbacks for this effect to improve performance when needed.
     * @param onCompiled Callback that will be called when the shader is compiled.
     * @param onError Callback that will be called if an error occurs during shader compilation.
     * @param indexParameters Parameters to be used with Babylons include syntax to iterate over an array (eg. {lights: 10})
     */
    constructor(baseName: any, attributesNamesOrOptions: string[] | IEffectCreationOptions, uniformsNamesOrEngine: string[] | ThinEngine, samplers: Nullable<string[]> = null,
        engine?: ThinEngine, defines: Nullable<string> = null,
        fallbacks: Nullable<IEffectFallbacks> = null, onCompiled: Nullable<(effect: Effect) => void> = null, onError: Nullable<(effect: Effect, errors: string) => void> = null, indexParameters?: any) {
        this.name = baseName;

        if ((<IEffectCreationOptions>attributesNamesOrOptions).attributes) {
            var options = <IEffectCreationOptions>attributesNamesOrOptions;
            this._engine = <Engine>uniformsNamesOrEngine;

            this._attributesNames = options.attributes;
            this._uniformsNames = options.uniformsNames.concat(options.samplers);
            this._samplerList = options.samplers.slice();
            this.defines = options.defines;
            this.onError = options.onError;
            this.onCompiled = options.onCompiled;
            this._fallbacks = options.fallbacks;
            this._indexParameters = options.indexParameters;
            this._transformFeedbackVaryings = options.transformFeedbackVaryings || null;

            if (options.uniformBuffersNames) {
                for (var i = 0; i < options.uniformBuffersNames.length; i++) {
                    this._uniformBuffersNames[options.uniformBuffersNames[i]] = i;
                }
            }
        } else {
            this._engine = <Engine>engine;
            this.defines = (defines == null ? "" : defines);
            this._uniformsNames = (<string[]>uniformsNamesOrEngine).concat(<string[]>samplers);
            this._samplerList = samplers ? <string[]>samplers.slice() : [];
            this._attributesNames = (<string[]>attributesNamesOrOptions);

            this.onError = onError;
            this.onCompiled = onCompiled;

            this._indexParameters = indexParameters;
            this._fallbacks = fallbacks;
        }

        this._attributeLocationByName = { };

        this.uniqueId = Effect._uniqueIdSeed++;

        var vertexSource: any;
        var fragmentSource: any;

        let hostDocument = DomManagement.IsWindowObjectExist() ? this._engine.getHostDocument() : null;

        if (baseName.vertexSource) {
            vertexSource = "source:" + baseName.vertexSource;
        } else if (baseName.vertexElement) {
            vertexSource = hostDocument ? hostDocument.getElementById(baseName.vertexElement) : null;

            if (!vertexSource) {
                vertexSource = baseName.vertexElement;
            }
        } else {
            vertexSource = baseName.vertex || baseName;
        }

        if (baseName.fragmentSource) {
            fragmentSource = "source:" + baseName.fragmentSource;
        } else if (baseName.fragmentElement) {
            fragmentSource = hostDocument ? hostDocument.getElementById(baseName.fragmentElement) : null;

            if (!fragmentSource) {
                fragmentSource = baseName.fragmentElement;
            }
        } else {
            fragmentSource = baseName.fragment || baseName;
        }

        let processorOptions = {
            defines: this.defines.split("\n"),
            indexParameters: this._indexParameters,
            isFragment: false,
            shouldUseHighPrecisionShader: this._engine._shouldUseHighPrecisionShader,
            processor: this._engine._shaderProcessor,
            supportsUniformBuffers: this._engine.supportsUniformBuffers,
            shadersRepository: Effect.ShadersRepository,
            includesShadersStore: Effect.IncludesShadersStore,
            version: (this._engine.webGLVersion * 100).toString(),
            platformName: this._engine.webGLVersion >= 2 ? "WEBGL2" : "WEBGL1"
        };

        this._loadShader(vertexSource, "Vertex", "", (vertexCode) => {
            this._loadShader(fragmentSource, "Fragment", "Pixel", (fragmentCode) => {
                ShaderProcessor.Process(vertexCode, processorOptions, (migratedVertexCode) => {
                    processorOptions.isFragment = true;
                    ShaderProcessor.Process(fragmentCode, processorOptions, (migratedFragmentCode) => {
                        this._useFinalCode(migratedVertexCode, migratedFragmentCode, baseName);
                    });
                });
            });
        });
    }

    private _useFinalCode(migratedVertexCode: string, migratedFragmentCode: string, baseName: any) {
        if (baseName) {
            var vertex = baseName.vertexElement || baseName.vertex || baseName.spectorName || baseName;
            var fragment = baseName.fragmentElement || baseName.fragment || baseName.spectorName || baseName;

            this._vertexSourceCode = "#define SHADER_NAME vertex:" + vertex + "\n" + migratedVertexCode;
            this._fragmentSourceCode = "#define SHADER_NAME fragment:" + fragment + "\n" + migratedFragmentCode;
        } else {
            this._vertexSourceCode = migratedVertexCode;
            this._fragmentSourceCode = migratedFragmentCode;
        }
        this._prepareEffect();
    }

    /**
     * Unique key for this effect
     */
    public get key(): string {
        return this._key;
    }

    /**
     * If the effect has been compiled and prepared.
     * @returns if the effect is compiled and prepared.
     */
    public isReady(): boolean {
        try {
            return this._isReadyInternal();
        }
        catch {
            return false;
        }
    }

    private _isReadyInternal(): boolean {
        if (this._isReady) {
            return true;
        }
        if (this._pipelineContext) {
            return this._pipelineContext.isReady;
        }
        return false;
    }

    /**
     * The engine the effect was initialized with.
     * @returns the engine.
     */
    public getEngine(): Engine {
        return this._engine;
    }

    /**
     * The pipeline context for this effect
     * @returns the associated pipeline context
     */
    public getPipelineContext(): Nullable<IPipelineContext> {
        return this._pipelineContext;
    }

    /**
     * The set of names of attribute variables for the shader.
     * @returns An array of attribute names.
     */
    public getAttributesNames(): string[] {
        return this._attributesNames;
    }

    /**
     * Returns the attribute at the given index.
     * @param index The index of the attribute.
     * @returns The location of the attribute.
     */
    public getAttributeLocation(index: number): number {
        return this._attributes[index];
    }

    /**
     * Returns the attribute based on the name of the variable.
     * @param name of the attribute to look up.
     * @returns the attribute location.
     */
    public getAttributeLocationByName(name: string): number {
        return this._attributeLocationByName[name];
    }

    /**
     * The number of attributes.
     * @returns the numnber of attributes.
     */
    public getAttributesCount(): number {
        return this._attributes.length;
    }

    /**
     * Gets the index of a uniform variable.
     * @param uniformName of the uniform to look up.
     * @returns the index.
     */
    public getUniformIndex(uniformName: string): number {
        return this._uniformsNames.indexOf(uniformName);
    }

    /**
     * Returns the attribute based on the name of the variable.
     * @param uniformName of the uniform to look up.
     * @returns the location of the uniform.
     */
    public getUniform(uniformName: string): Nullable<WebGLUniformLocation> {
        return this._uniforms[uniformName];
    }

    /**
     * Returns an array of sampler variable names
     * @returns The array of sampler variable neames.
     */
    public getSamplers(): string[] {
        return this._samplerList;
    }

    /**
     * The error from the last compilation.
     * @returns the error string.
     */
    public getCompilationError(): string {
        return this._compilationError;
    }

    /**
     * Gets a boolean indicating that all fallbacks were used during compilation
     * @returns true if all fallbacks were used
     */
    public allFallbacksProcessed(): boolean {
        return this._allFallbacksProcessed;
    }

    /**
     * Adds a callback to the onCompiled observable and call the callback imediatly if already ready.
     * @param func The callback to be used.
     */
    public executeWhenCompiled(func: (effect: Effect) => void): void {
        if (this.isReady()) {
            func(this);
            return;
        }

        this.onCompileObservable.add((effect) => {
            func(effect);
        });

        if (!this._pipelineContext || this._pipelineContext.isAsync) {
            setTimeout(() => {
                this._checkIsReady(null);
            }, 16);
        }
    }

    private _checkIsReady(previousPipelineContext: Nullable<IPipelineContext>) {
        try {
            if (this._isReadyInternal()) {
                return;
            }
        } catch (e) {
            this._processCompilationErrors(e, previousPipelineContext);
            return;
        }

        setTimeout(() => {
            this._checkIsReady(previousPipelineContext);
        }, 16);
    }

    private _loadShader(shader: any, key: string, optionalKey: string, callback: (data: any) => void): void {
        if (typeof(HTMLElement) !== "undefined") {
            // DOM element ?
            if (shader instanceof HTMLElement) {
                var shaderCode = DomManagement.GetDOMTextContent(shader);
                callback(shaderCode);
                return;
            }
        }

        // Direct source ?
        if (shader.substr(0, 7) === "source:") {
            callback(shader.substr(7));
            return;
        }

        // Base64 encoded ?
        if (shader.substr(0, 7) === "base64:") {
            var shaderBinary = window.atob(shader.substr(7));
            callback(shaderBinary);
            return;
        }

        // Is in local store ?
        if (Effect.ShadersStore[shader + key + "Shader"]) {
            callback(Effect.ShadersStore[shader + key + "Shader"]);
            return;
        }

        if (optionalKey && Effect.ShadersStore[shader + optionalKey + "Shader"]) {
            callback(Effect.ShadersStore[shader + optionalKey + "Shader"]);
            return;
        }

        var shaderUrl;

        if (shader[0] === "." || shader[0] === "/" || shader.indexOf("http") > -1) {
            shaderUrl = shader;
        } else {
            shaderUrl = Effect.ShadersRepository + shader;
        }

        // Vertex shader
        this._engine._loadFile(shaderUrl + "." + key.toLowerCase() + ".fx", callback);
    }

    /**
     * Recompiles the webGL program
     * @param vertexSourceCode The source code for the vertex shader.
     * @param fragmentSourceCode The source code for the fragment shader.
     * @param onCompiled Callback called when completed.
     * @param onError Callback called on error.
     * @hidden
     */
    public _rebuildProgram(vertexSourceCode: string, fragmentSourceCode: string, onCompiled: (pipelineContext: IPipelineContext) => void, onError: (message: string) => void) {
        this._isReady = false;

        this._vertexSourceCodeOverride = vertexSourceCode;
        this._fragmentSourceCodeOverride = fragmentSourceCode;
        this.onError = (effect, error) => {
            if (onError) {
                onError(error);
            }
        };
        this.onCompiled = () => {
            var scenes = this.getEngine().scenes;
            if (scenes) {
                for (var i = 0; i < scenes.length; i++) {
                    scenes[i].markAllMaterialsAsDirty(Constants.MATERIAL_AllDirtyFlag);
                }
            }

            this._pipelineContext!._handlesSpectorRebuildCallback(onCompiled);
        };
        this._fallbacks = null;
        this._prepareEffect();
    }

    /**
     * Prepares the effect
     * @hidden
     */
    public _prepareEffect() {
        let attributesNames = this._attributesNames;
        let defines = this.defines;
        this._valueCache = {};

        var previousPipelineContext = this._pipelineContext;

        try {
            let engine = this._engine;

            this._pipelineContext = engine.createPipelineContext();

            let rebuildRebind = this._rebuildProgram.bind(this);
            if (this._vertexSourceCodeOverride && this._fragmentSourceCodeOverride) {
                engine._preparePipelineContext(this._pipelineContext, this._vertexSourceCodeOverride, this._fragmentSourceCodeOverride, true, rebuildRebind, null, this._transformFeedbackVaryings);
            }
            else {
                engine._preparePipelineContext(this._pipelineContext, this._vertexSourceCode, this._fragmentSourceCode, false, rebuildRebind, defines, this._transformFeedbackVaryings);
            }

            engine._executeWhenRenderingStateIsCompiled(this._pipelineContext, () => {
                if (engine.supportsUniformBuffers) {
                    for (var name in this._uniformBuffersNames) {
                        this.bindUniformBlock(name, this._uniformBuffersNames[name]);
                    }
                }

                let uniforms = engine.getUniforms(this._pipelineContext!, this._uniformsNames);
                uniforms.forEach((uniform, index) => {
                    this._uniforms[this._uniformsNames[index]] = uniform;
                });

                this._attributes = engine.getAttributes(this._pipelineContext!, attributesNames);
                if (attributesNames) {
                    for (let i = 0; i < attributesNames.length; i++) {
                        const name = attributesNames[i];
                        this._attributeLocationByName[name] = this._attributes[i];
                    }
                }

                var index: number;
                for (index = 0; index < this._samplerList.length; index++) {
                    var sampler = this.getUniform(this._samplerList[index]);

                    if (sampler == null) {
                        this._samplerList.splice(index, 1);
                        index--;
                    }
                }

                this._samplerList.forEach((name, index) => {
                    this._samplers[name] = index;
                });

                engine.bindSamplers(this);

                this._compilationError = "";
                this._isReady = true;
                if (this.onCompiled) {
                    this.onCompiled(this);
                }
                this.onCompileObservable.notifyObservers(this);
                this.onCompileObservable.clear();

                // Unbind mesh reference in fallbacks
                if (this._fallbacks) {
                    this._fallbacks.unBindMesh();
                }

                if (previousPipelineContext) {
                    this.getEngine()._deletePipelineContext(previousPipelineContext);
                }
            });

            if (this._pipelineContext.isAsync) {
                this._checkIsReady(previousPipelineContext);
            }

        } catch (e) {
           this._processCompilationErrors(e, previousPipelineContext);
        }
    }

    private _getShaderCodeAndErrorLine(shader: WebGLShader, error: Nullable<string>, isFragment: boolean): [Nullable<string>, Nullable<string>] {
        const regexp = isFragment ? /FRAGMENT SHADER ERROR: 0:(\d+?):/ : /VERTEX SHADER ERROR: 0:(\d+?):/;
        const code = this._engine._getShaderSource(shader);

        let errorLine = null;

        if (error && code) {
            const res = error.match(regexp);
            if (res && res.length === 2) {
                const lineNumber = parseInt(res[1]);
                const lines = code.split("\n", -1);
                if (lines.length >= lineNumber) {
                    errorLine = `Offending line [${lineNumber}] in ${isFragment ? "fragment" : "vertex"} code: ${lines[lineNumber - 1]}`;
                }
            }
        }

        return [code, errorLine];
    }

    private _isWebGLPipelineContext(obj: any): obj is WebGLPipelineContext {
        return obj.vertexShader && obj.fragmentShader;
    }

    private _processCompilationErrors(e: any, previousPipelineContext: Nullable<IPipelineContext> = null) {
        this._compilationError = e.message;
        let attributesNames = this._attributesNames;
        let fallbacks = this._fallbacks;

        // Let's go through fallbacks then
        Logger.Error("Unable to compile effect:");
        Logger.Error("Uniforms: " + this._uniformsNames.map(function(uniform) {
            return " " + uniform;
        }));
        Logger.Error("Attributes: " + attributesNames.map(function(attribute) {
            return " " + attribute;
        }));
        Logger.Error("Defines:\r\n" + this.defines);
        if (Effect.LogShaderCodeOnCompilationError && this._isWebGLPipelineContext(this._pipelineContext)) {
            let pcw = this._pipelineContext;
            let lineErrorVertex = null, lineErrorFragment = null, code = null;
            if (pcw.vertexShader) {
                [code, lineErrorVertex] = this._getShaderCodeAndErrorLine(pcw.vertexShader, this._compilationError, false);
                if (code) {
                    Logger.Error("Vertex code:");
                    Logger.Error(code);
                }
            }
            if (pcw.fragmentShader) {
                [code, lineErrorFragment] = this._getShaderCodeAndErrorLine(pcw.fragmentShader, this._compilationError, true);
                if (code) {
                    Logger.Error("Fragment code:");
                    Logger.Error(code);
                }
            }
            if (lineErrorVertex) {
                Logger.Error(lineErrorVertex);
            }
            if (lineErrorFragment) {
                Logger.Error(lineErrorFragment);
            }
        }
        Logger.Error("Error: " + this._compilationError);
{'fraction_non_alphanumeric': 0.1024652551252397, 'fraction_numerical': 0.0018187928750766067, 'mean_word_length': 2.195048004042446, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 129, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6055293', 'n_tokens_mistral': 13310, 'n_tokens_neox': 12897, 'n_words': 3417}