<commit_before>package cdb;
import cdb.Data;

enum LocField {
	LName( c : Column );
	LSub( c : Column, s : Sheet, e : Array<LocField> );
	LSingle( c : Column, e : LocField );
}

class Lang {

	var root : Data;

	public function new(root) {
		this.root = root;
	}

	public dynamic function onMissing( s : String ) {
		trace(s);
	}

	public function getSub( s : Sheet, c : Column ) {
		return getSheet(s.name + "@" + c.name);
	}

	function getSheet( name : String ) {
		for( s in root.sheets )
			if( s.name == name )
				return s;
		return null;
	}

	function makeLocField(c:Column, s:Sheet) {
		switch( c.type ) {
		case TString if( c.kind == Localizable ):
			return LName(c);
		case TList, TProperties:
			var ssub = getSub(s,c);
			var fl = makeSheetFields(ssub);
			if( fl.length == 0 )
				return null;
			return LSub(c, ssub, fl);
		default:
			return null;
		}
	}

	function makeSheetFields(s:Sheet) : Array<LocField> {
		var fields = [];
		for( c in s.columns ) {
			var f = makeLocField(c, s);
			if( f != null )
				switch( f ) {
				case LSub(c, _, fl) if( c.type == TProperties ):
					for( f in fl )
						fields.push(LSingle(c, f));
				default:
					fields.push(f);
				}
		}
		return fields;
	}

	public function apply( xml : String ) {
		var x = Xml.parse(xml).firstElement();
		var xsheets = new Map();
		for( e in x.elements() )
			xsheets.set(e.get("name"), e);
		for( s in root.sheets ) {
			if( s.props.hide ) continue;
			var x = xsheets.get(s.name);
			if( x == null ) continue;

			var path = [s.name];
			applySheet(path, s, makeSheetFields(s), s.lines, x);
		}
	}

	function applySheet( path : Array<String>, s : Sheet, fields : Array<LocField>, objects : Array<Dynamic>, x : Xml ) {
		var idField = null;
		for( c in s.columns )
			if( c.type == TId ) {
				idField = c.name;
				break;
			}

		if( idField == null ) {

			var byIndex = [];
			if( x != null )
				for( e in x.elements() ) {
					var m = new Map();
					for( e in e.elements() )
						m.set(e.nodeName, e);
					byIndex[Std.parseInt(e.nodeName)] = m;
				}

			for( i in 0...objects.length )
				for( f in fields ) {
					path.push("["+i+"]");
					applyRec(path, f, objects[i], byIndex[i]);
					path.pop();
				}

		} else {

			var byID = new Map();
			if( x != null )
				for( e in x.elements() ) {
					var m = new Map();
					for( e in e.elements() )
						m.set(e.nodeName, e);
					byID.set(e.nodeName, m);
				}

			for( o in objects )
				for( f in fields ) {
					var id = Reflect.field(o, idField);
					path.push(id);
					applyRec(path, f, o, byID.get(id));
					path.pop();
				}
		}
	}

	function applyRec( path : Array<String>, f : LocField, o : Dynamic, data : Map<String,Xml> ) {
		switch( f ) {
		case LName(c):
			var v = data == null ? null : data.get(c.name);
			if( v != null )
				Reflect.setField(o, c.name, new haxe.xml.Fast(v).innerHTML);
			else {
				var v = Reflect.field(o, c.name);
				if( v != null && v != "" ) {
					path.push(c.name);
					onMissing("Missing " + path.join("."));
					path.pop();
				}
			}
		case LSingle(c, f):<commit_msg>fixed html unescaping<commit_after>package cdb;
import cdb.Data;

enum LocField {
	LName( c : Column );
	LSub( c : Column, s : Sheet, e : Array<LocField> );
	LSingle( c : Column, e : LocField );
}

class Lang {

	var root : Data;

	public function new(root) {
		this.root = root;
	}

	public dynamic function onMissing( s : String ) {
		trace(s);
	}

	public function getSub( s : Sheet, c : Column ) {
		return getSheet(s.name + "@" + c.name);
	}

	function getSheet( name : String ) {
		for( s in root.sheets )
			if( s.name == name )
				return s;
		return null;
	}

	function makeLocField(c:Column, s:Sheet) {
		switch( c.type ) {
		case TString if( c.kind == Localizable ):
			return LName(c);
		case TList, TProperties:
			var ssub = getSub(s,c);
			var fl = makeSheetFields(ssub);
			if( fl.length == 0 )
				return null;
			return LSub(c, ssub, fl);
		default:
			return null;
		}
	}

	function makeSheetFields(s:Sheet) : Array<LocField> {
		var fields = [];
		for( c in s.columns ) {
			var f = makeLocField(c, s);
			if( f != null )
				switch( f ) {
				case LSub(c, _, fl) if( c.type == TProperties ):
					for( f in fl )
						fields.push(LSingle(c, f));
				default:
					fields.push(f);
				}
		}
		return fields;
	}

	public function apply( xml : String ) {
		var x = Xml.parse(xml).firstElement();
		var xsheets = new Map();
		for( e in x.elements() )
			xsheets.set(e.get("name"), e);
		for( s in root.sheets ) {
			if( s.props.hide ) continue;
			var x = xsheets.get(s.name);
			if( x == null ) continue;

			var path = [s.name];
			applySheet(path, s, makeSheetFields(s), s.lines, x);
		}
	}

	function applySheet( path : Array<String>, s : Sheet, fields : Array<LocField>, objects : Array<Dynamic>, x : Xml ) {
		var idField = null;
		for( c in s.columns )
			if( c.type == TId ) {
				idField = c.name;
				break;
			}

		if( idField == null ) {

			var byIndex = [];
			if( x != null )
				for( e in x.elements() ) {
					var m = new Map();
					for( e in e.elements() )
						m.set(e.nodeName, e);
					byIndex[Std.parseInt(e.nodeName)] = m;
				}

			for( i in 0...objects.length )
				for( f in fields ) {
					path.push("["+i+"]");
					applyRec(path, f, objects[i], byIndex[i]);
					path.pop();
				}

		} else {

			var byID = new Map();
			if( x != null )
				for( e in x.elements() ) {
					var m = new Map();
					for( e in e.elements() )
						m.set(e.nodeName, e);
					byID.set(e.nodeName, m);
				}

			for( o in objects )
				for( f in fields ) {
					var id = Reflect.field(o, idField);
					path.push(id);
					applyRec(path, f, o, byID.get(id));
					path.pop();
				}
		}
	}

	function applyRec( path : Array<String>, f : LocField, o : Dynamic, data : Map<String,Xml> ) {
		switch( f ) {
		case LName(c):
			var v = data == null ? null : data.get(c.name);
			if( v != null )
				Reflect.setField(o, c.name, StringTools.htmlUnescape(new haxe.xml.Fast(v).innerHTML));
			else {
				var v = Reflect.field(o, c.name);
				if( v != null && v != "" ) {
					path.push(c.name);
					onMissing("Missing " + path.join("."));
					path.pop();
				}
			}
		case LSingle(c, f):
{'fraction_non_alphanumeric': 0.1691485917772742, 'fraction_numerical': 0.0006474587245063128, 'mean_word_length': 2.4365962180200222, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6758549', 'n_tokens_mistral': 2863, 'n_tokens_neox': 2355, 'n_words': 637}