<commit_before>package hxsl;
using hxsl.Ast;
import hxsl.RuntimeShader;

class SearchMap {
	public var linked : RuntimeShader;
	public var next : Map<Int,SearchMap>;
	public function new() {
	}
}

class Cache {

	#if shader_debug_dump
	public static var DEBUG_IDS = false;
	public static var TRACE = false;
	#end

	var linkCache : SearchMap;
	var linkShaders : Map<String, Shader>;
	var byID : Map<String, RuntimeShader>;
	public var constsToGlobal : Bool;

	function new() {
		constsToGlobal = false;
		linkCache = new SearchMap();
		linkShaders = new Map();
		byID = new Map();
	}

	/**
		Creates a shader that generate the output requested.
	**/
	public function getLinkShader( vars : Array<Output> ) {
		var key = vars.join(",");
		var shader = linkShaders.get(key);
		if( shader != null )
			return shader;
		var s = new hxsl.SharedShader("");
		var v = vars.copy();
		var id = haxe.crypto.Md5.encode(vars.join(":")).substr(0, 8);
		s.data = {
			name : "shaderLinker_"+id,
			vars : [],
			funs : [],
		};
		var pos = null;
		var outVars = new Map<String,TVar>();
		var outputCount = 0;
		var tvec4 = TVec(4, VFloat);
		function makeVec( g, size, args : Array<Output>, makeOutExpr : Output -> Int -> TExpr ) {
			var out = [];
			var rem = size;
			for( i in 0...args.length ) {
				var e = makeOutExpr(args[args.length - 1 - i], rem - (args.length - 1 - i));
				rem -= Tools.size(e.t);
				out.unshift(e);
			}
			return { e : TCall({ e : TGlobal(g), t : TVoid, p : pos }, out), t : TVec(size,VFloat), p : pos };
		}
		function makeVar( name : String, t, parent : TVar ) {
			var path = parent == null ? name : parent.getName() + "." + name;
			var v = outVars.get(path);
			if( v != null )
				return v;
			v = {
				id : Tools.allocVarId(),
				name : name,
				type : t,
				kind : Var,
				parent : parent,
			};
			if( parent == null )
				s.data.vars.push(v);
			else {
				switch( parent.type ) {
				case TStruct(vl): vl.push(v);
				default: throw "assert";
				}
			}
			outVars.set(path, v);
			return v;
		}

		function makeOutExpr( v : Output, rem : Int ) : TExpr {
			switch( v ) {
			case Const(v):
				return { e : TConst(CFloat(v)), t : TFloat, p : pos };
			case Vec2(args):
				return makeVec(Vec2, 2, args, makeOutExpr);
			case Vec3(args):
				return makeVec(Vec3, 3, args, makeOutExpr);
			case Vec4(args):
				return makeVec(Vec4, 4, args, makeOutExpr);
			case Value(vname,size):
				var v = outVars.get(vname);
				if( v != null )
					return { e : TVar(v), t : v.type, p : pos };
				var path = vname.split(".");
				var parent : TVar = null;
				while( path.length > 1 )
					parent = makeVar(path.shift(), TStruct([]), parent);
				if( size != null )
					rem = size;
				v = makeVar(path.shift(), rem == 1 ? TFloat : TVec(rem, VFloat), parent);
				return { e : TVar(v), t : v.type, p : pos };
			case PackNormal(v):
				return { e : TCall({ e : TGlobal(PackNormal), t : TVoid, p : pos }, [makeOutExpr(v,3)]), t : tvec4, p : pos };
			case PackFloat(v):
				return { e : TCall({ e : TGlobal(Pack), t : TVoid, p : pos }, [makeOutExpr(v, 1)]), t : tvec4, p : pos };
			case Swiz(v, comps):
				return { e : TSwiz(makeOutExpr(v,4), comps), t : TVec(comps.length, VFloat), p : pos };
			}
		}
		function makeOutput( v : Output ) : TExpr {
			var ov : TVar = {
				id : Tools.allocVarId(),
				type : tvec4,
				name : "OUTPUT" + (outputCount++),
				kind : Output,
			};
			s.data.vars.push(ov);
			return { e : TBinop(OpAssign,{ e : TVar(ov), t : tvec4, p : pos }, makeOutExpr(v,4)), t : TVoid, p : pos };
		}
		function defineFun( kind : FunctionKind, vars : Array<Output> ) {
			var fv : TVar = {
				id : Tools.allocVarId(),
				type : TFun([]),
				name : ("" + kind).toLowerCase(),
				kind : Function,
			};
			var f : TFunction = {
				kind : kind,
				ref : fv,
				args : [],
				ret : TVoid,
				expr : { e : TBlock([for( v in vars ) makeOutput(v)]), p : pos, t : TVoid },
			};
			s.data.funs.push(f);
		}
		defineFun(Vertex, [Value("output.position")]);
		defineFun(Fragment, vars);

		shader = std.Type.createEmptyInstance(Shader);
		@:privateAccess shader.shader = s;
		linkShaders.set(key, shader);
		@:privateAccess shader.updateConstantsFinal(null);

		return shader;
	}

	@:noDebug
	public function link( shaders : hxsl.ShaderList ) {
		var c = linkCache;
		for( s in shaders ) {
			var i = @:privateAccess s.instance;
			if( c.next == null ) c.next = new Map();
			var cs = c.next.get(i.id);
			if( cs == null ) {
				cs = new SearchMap();
				c.next.set(i.id, cs);
			}
			c = cs;
		}
		if( c.linked == null )
			c.linked = compileRuntimeShader(shaders);
		return c.linked;
	}

	function compileRuntimeShader( shaders : hxsl.ShaderList ) {
		var shaderDatas = [];
		var index = 0;
		for( s in shaders ) {
			var i = @:privateAccess s.instance;
			shaderDatas.push( { inst : i, index : index++ } );
		}
		shaderDatas.reverse(); // default is reverse order

		#if debug
		for( s in shaderDatas ) Printer.check(s.inst.shader);
		#end

		#if shader_debug_dump
		var shaderId = @:privateAccess RuntimeShader.UID;
		if( shaderId == 0 ) try sys.FileSystem.createDirectory("shaders") catch( e : Dynamic ) {};
		var dbg = sys.io.File.write("shaders/"+shaderId+"_dump.c");
		if( dbg != null ) {
			dbg.writeString("----- DATAS ----\n\n");
			for( s in shaderDatas ) {
				dbg.writeString("\t\t**** " + s.inst.shader.name + (s.p == 0 ? "" : " P="+s.p)+ " *****\n");
				dbg.writeString(Printer.shaderToString(s.inst.shader,DEBUG_IDS)+"\n\n");
			}
		}
		//TRACE = shaderId == 0;
		#end

		var linker = new hxsl.Linker();
		var s = linker.link([for( s in shaderDatas ) s.inst.shader]);

		#if debug
		Printer.check(s,[for( s in shaderDatas ) s.inst.shader]);
		#end

		#if shader_debug_dump
		if( dbg != null ) {
			dbg.writeString("----- LINK ----\n\n");
			dbg.writeString(Printer.shaderToString(s,DEBUG_IDS)+"\n\n");
		}<commit_msg>keep sorting shaders at runtime<commit_after>package hxsl;
using hxsl.Ast;
import hxsl.RuntimeShader;

class SearchMap {
	public var linked : RuntimeShader;
	public var next : Map<Int,SearchMap>;
	public function new() {
	}
}

class Cache {

	#if shader_debug_dump
	public static var DEBUG_IDS = false;
	public static var TRACE = false;
	#end

	var linkCache : SearchMap;
	var linkShaders : Map<String, Shader>;
	var byID : Map<String, RuntimeShader>;
	public var constsToGlobal : Bool;

	function new() {
		constsToGlobal = false;
		linkCache = new SearchMap();
		linkShaders = new Map();
		byID = new Map();
	}

	/**
		Creates a shader that generate the output requested.
	**/
	public function getLinkShader( vars : Array<Output> ) {
		var key = vars.join(",");
		var shader = linkShaders.get(key);
		if( shader != null )
			return shader;
		var s = new hxsl.SharedShader("");
		var v = vars.copy();
		var id = haxe.crypto.Md5.encode(vars.join(":")).substr(0, 8);
		s.data = {
			name : "shaderLinker_"+id,
			vars : [],
			funs : [],
		};
		var pos = null;
		var outVars = new Map<String,TVar>();
		var outputCount = 0;
		var tvec4 = TVec(4, VFloat);
		function makeVec( g, size, args : Array<Output>, makeOutExpr : Output -> Int -> TExpr ) {
			var out = [];
			var rem = size;
			for( i in 0...args.length ) {
				var e = makeOutExpr(args[args.length - 1 - i], rem - (args.length - 1 - i));
				rem -= Tools.size(e.t);
				out.unshift(e);
			}
			return { e : TCall({ e : TGlobal(g), t : TVoid, p : pos }, out), t : TVec(size,VFloat), p : pos };
		}
		function makeVar( name : String, t, parent : TVar ) {
			var path = parent == null ? name : parent.getName() + "." + name;
			var v = outVars.get(path);
			if( v != null )
				return v;
			v = {
				id : Tools.allocVarId(),
				name : name,
				type : t,
				kind : Var,
				parent : parent,
			};
			if( parent == null )
				s.data.vars.push(v);
			else {
				switch( parent.type ) {
				case TStruct(vl): vl.push(v);
				default: throw "assert";
				}
			}
			outVars.set(path, v);
			return v;
		}

		function makeOutExpr( v : Output, rem : Int ) : TExpr {
			switch( v ) {
			case Const(v):
				return { e : TConst(CFloat(v)), t : TFloat, p : pos };
			case Vec2(args):
				return makeVec(Vec2, 2, args, makeOutExpr);
			case Vec3(args):
				return makeVec(Vec3, 3, args, makeOutExpr);
			case Vec4(args):
				return makeVec(Vec4, 4, args, makeOutExpr);
			case Value(vname,size):
				var v = outVars.get(vname);
				if( v != null )
					return { e : TVar(v), t : v.type, p : pos };
				var path = vname.split(".");
				var parent : TVar = null;
				while( path.length > 1 )
					parent = makeVar(path.shift(), TStruct([]), parent);
				if( size != null )
					rem = size;
				v = makeVar(path.shift(), rem == 1 ? TFloat : TVec(rem, VFloat), parent);
				return { e : TVar(v), t : v.type, p : pos };
			case PackNormal(v):
				return { e : TCall({ e : TGlobal(PackNormal), t : TVoid, p : pos }, [makeOutExpr(v,3)]), t : tvec4, p : pos };
			case PackFloat(v):
				return { e : TCall({ e : TGlobal(Pack), t : TVoid, p : pos }, [makeOutExpr(v, 1)]), t : tvec4, p : pos };
			case Swiz(v, comps):
				return { e : TSwiz(makeOutExpr(v,4), comps), t : TVec(comps.length, VFloat), p : pos };
			}
		}
		function makeOutput( v : Output ) : TExpr {
			var ov : TVar = {
				id : Tools.allocVarId(),
				type : tvec4,
				name : "OUTPUT" + (outputCount++),
				kind : Output,
			};
			s.data.vars.push(ov);
			return { e : TBinop(OpAssign,{ e : TVar(ov), t : tvec4, p : pos }, makeOutExpr(v,4)), t : TVoid, p : pos };
		}
		function defineFun( kind : FunctionKind, vars : Array<Output> ) {
			var fv : TVar = {
				id : Tools.allocVarId(),
				type : TFun([]),
				name : ("" + kind).toLowerCase(),
				kind : Function,
			};
			var f : TFunction = {
				kind : kind,
				ref : fv,
				args : [],
				ret : TVoid,
				expr : { e : TBlock([for( v in vars ) makeOutput(v)]), p : pos, t : TVoid },
			};
			s.data.funs.push(f);
		}
		defineFun(Vertex, [Value("output.position")]);
		defineFun(Fragment, vars);

		shader = std.Type.createEmptyInstance(Shader);
		@:privateAccess shader.shader = s;
		linkShaders.set(key, shader);
		@:privateAccess shader.updateConstantsFinal(null);

		return shader;
	}

	@:noDebug
	public function link( shaders : hxsl.ShaderList ) {
		var c = linkCache;
		for( s in shaders ) {
			var i = @:privateAccess s.instance;
			if( c.next == null ) c.next = new Map();
			var cs = c.next.get(i.id);
			if( cs == null ) {
				cs = new SearchMap();
				c.next.set(i.id, cs);
			}
			c = cs;
		}
		if( c.linked == null )
			c.linked = compileRuntimeShader(shaders);
		return c.linked;
	}

	function compileRuntimeShader( shaders : hxsl.ShaderList ) {
		var shaderDatas = [];
		var index = 0;
		for( s in shaders ) {
			var i = @:privateAccess s.instance;
			shaderDatas.push( { inst : i, p : s.priority, index : index++ } );
		}
		shaderDatas.reverse(); // default is reverse order
		/*
			Our shader list is supposedly already sorted. However some shaders
			with high priority might be prepended at later stage (eg: Base2D)
			So let's sort again just in case.
		*/
		haxe.ds.ArraySort.sort(shaderDatas, function(s1, s2) return s2.p - s1.p);

		#if debug
		for( s in shaderDatas ) Printer.check(s.inst.shader);
		#end

		#if shader_debug_dump
		var shaderId = @:privateAccess RuntimeShader.UID;
		if( shaderId == 0 ) try sys.FileSystem.createDirectory("shaders") catch( e : Dynamic ) {};
		var dbg = sys.io.File.write("shaders/"+shaderId+"_dump.c");
		if( dbg != null ) {
			dbg.writeString("----- DATAS ----\n\n");
			for( s in shaderDatas ) {
				dbg.writeString("\t\t**** " + s.inst.shader.name + (s.p == 0 ? "" : " P="+s.p)+ " *****\n");
				dbg.writeString(Printer.shaderToString(s.inst.shader,DEBUG_IDS)+"\n\n");
			}
		}
		//TRACE = shaderId == 0;
		#end

		var linker = new hxsl.Linker();
		var s = linker.link([for( s in shaderDatas ) s.inst.shader]);

		#if debug
		Printer.check(s,[for( s in shaderDatas ) s.inst.shader]);
		#end

		#if shader_debug_dump
		if( dbg != null ) {
			dbg.writeString("----- LINK ----\n\n");
			dbg.writeString(Printer.shaderToString(s,DEBUG_IDS)+"\n\n");
		}
{'fraction_non_alphanumeric': 0.17012448132780084, 'fraction_numerical': 0.005726141078838174, 'mean_word_length': 2.839120739088882, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3043708', 'n_tokens_mistral': 5206, 'n_tokens_neox': 4469, 'n_words': 1188}