<commit_before>﻿using LSLib.LS;
using LSLib.LS.Story;
using LSLib.LS.Story.Compiler;
using LSLib.LS.Story.GoalParser;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

namespace LSTools.StoryCompiler
{
    class ModCompiler
    {
        class GoalScript
        {
            public string Name;
            public string Path;
            public byte[] ScriptBody;
        }

        private Logger Logger;
        private String GameDataPath;
        private Compiler Compiler = new Compiler();
        private ModResources Mods = new ModResources();
        private List<GoalScript> GoalScripts = new List<GoalScript>();
        private List<byte[]> GameObjectLSFs = new List<byte[]>();

        public bool CheckOnly = false;
        public bool CheckGameObjects = false;

        public ModCompiler(Logger logger, String gameDataPath)
        {
            Logger = logger;
            GameDataPath = gameDataPath;
        }

        private void LoadStoryHeaders(Stream stream)
        {
            var hdrLoader = new StoryHeaderLoader(Compiler.Context);
            var declarations = hdrLoader.ParseHeader(stream);
            if (declarations == null)
            {
                throw new Exception("Failed to parse story header file");
            }

            hdrLoader.LoadHeader(declarations);
        }
        
        public void SetWarningOptions(Dictionary<string, bool> options)
        {
            foreach (var option in options)
            {
                Compiler.Context.Log.WarningSwitches[option.Key] = option.Value;
            }
        }

        class IRBuildTasks
        {
            public ConcurrentQueue<GoalScript> Inputs = new ConcurrentQueue<GoalScript>();
            public ConcurrentQueue<IRGoal> IRs = new ConcurrentQueue<IRGoal>();
        }

        private void BuildIR(IRBuildTasks tasks)
        {
            var goalLoader = new IRGenerator(Compiler.Context);
            while (tasks.Inputs.TryDequeue(out GoalScript script))
            {
                using (var stream = new MemoryStream(script.ScriptBody))
                {
                    var ast = goalLoader.ParseGoal(script.Path, stream);
                    var ir = goalLoader.GenerateGoalIR(ast);
                    ir.Name = script.Name;
                    tasks.IRs.Enqueue(ir);
                }
            }
        }

        private List<IRGoal> ParallelBuildIR()
        {
            var tasks = new IRBuildTasks();
            foreach (var script in GoalScripts)
            {
                tasks.Inputs.Enqueue(script);
            }

            IRBuildTasks[] threadTasks = new[] { tasks, tasks, tasks, tasks };
            Task.WhenAll(threadTasks.Select(task => Task.Run(() => { BuildIR(task); }))).Wait();

            var sorted = new SortedDictionary<string, IRGoal>();
            while (tasks.IRs.TryDequeue(out IRGoal goal))
            {
                sorted[goal.Name] = goal;
            }

            return sorted.Values.ToList();
        }

        private void LoadGameObjects(Resource resource)
        {
            if (!resource.Regions.TryGetValue("Templates", out Region templates))
            {
                // TODO - log error
                return;
            }

            if (!templates.Children.TryGetValue("GameObjects", out List<LSLib.LS.Node> gameObjects))
            {
                // TODO - log error
                return;
            }

            foreach (var gameObject in gameObjects)
            {
                if (gameObject.Attributes.TryGetValue("MapKey", out NodeAttribute objectGuid)
                    && gameObject.Attributes.TryGetValue("Name", out NodeAttribute objectName)
                    && gameObject.Attributes.TryGetValue("Type", out NodeAttribute objectType))
                {
                    LSLib.LS.Story.Compiler.ValueType type = null;
                    switch ((string)objectType.Value)
                    {
                        case "item": type = Compiler.Context.LookupType("ITEMGUID"); break;
                        case "character": type = Compiler.Context.LookupType("CHARACTERGUID"); break;
                        case "trigger": type = Compiler.Context.LookupType("TRIGGERGUID"); break;
                        default:
                            // TODO - log unknown type
                            break;
                    }

                    if (type != null)
                    {
                        var gameObjectInfo = new GameObjectInfo
                        {
                            Name = objectName.Value + "_" + objectGuid.Value,
                            Type = type
                        };
                        Compiler.Context.GameObjects[(string)objectGuid.Value] = gameObjectInfo;
                    }
                }
            }
        }

        private void LoadGlobals()
        {
            foreach (var lsf in GameObjectLSFs)
            {
                using (var stream = new MemoryStream(lsf))
                using (var reader = new LSFReader(stream))
                {
                    var resource = reader.Read();
                    LoadGameObjects(resource);
                }
            }
        }

        private void LoadGoals(ModInfo mod)
        {
            foreach (var file in mod.Scripts)
            {
                var scriptStream = file.Value.MakeStream();
                try
                {
                    using (var reader = new BinaryReader(scriptStream))
                    {
                        var script = new GoalScript
                        {
                            Name = Path.GetFileNameWithoutExtension(file.Value.Name),
                            Path = file.Key,
                            ScriptBody = reader.ReadBytes((int)scriptStream.Length)
                        };
                        GoalScripts.Add(script);
                    }
                }<commit_msg>Fix file paths in debug info<commit_after>﻿using LSLib.LS;
using LSLib.LS.Story;
using LSLib.LS.Story.Compiler;
using LSLib.LS.Story.GoalParser;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

namespace LSTools.StoryCompiler
{
    class ModCompiler
    {
        class GoalScript
        {
            public string Name;
            public string Path;
            public byte[] ScriptBody;
        }

        private Logger Logger;
        private String GameDataPath;
        private Compiler Compiler = new Compiler();
        private ModResources Mods = new ModResources();
        private List<GoalScript> GoalScripts = new List<GoalScript>();
        private List<byte[]> GameObjectLSFs = new List<byte[]>();

        public bool CheckOnly = false;
        public bool CheckGameObjects = false;

        public ModCompiler(Logger logger, String gameDataPath)
        {
            Logger = logger;
            GameDataPath = gameDataPath;
        }

        private void LoadStoryHeaders(Stream stream)
        {
            var hdrLoader = new StoryHeaderLoader(Compiler.Context);
            var declarations = hdrLoader.ParseHeader(stream);
            if (declarations == null)
            {
                throw new Exception("Failed to parse story header file");
            }

            hdrLoader.LoadHeader(declarations);
        }
        
        public void SetWarningOptions(Dictionary<string, bool> options)
        {
            foreach (var option in options)
            {
                Compiler.Context.Log.WarningSwitches[option.Key] = option.Value;
            }
        }

        class IRBuildTasks
        {
            public ConcurrentQueue<GoalScript> Inputs = new ConcurrentQueue<GoalScript>();
            public ConcurrentQueue<IRGoal> IRs = new ConcurrentQueue<IRGoal>();
        }

        private void BuildIR(IRBuildTasks tasks)
        {
            var goalLoader = new IRGenerator(Compiler.Context);
            while (tasks.Inputs.TryDequeue(out GoalScript script))
            {
                using (var stream = new MemoryStream(script.ScriptBody))
                {
                    var ast = goalLoader.ParseGoal(script.Path, stream);
                    var ir = goalLoader.GenerateGoalIR(ast);
                    ir.Name = script.Name;
                    tasks.IRs.Enqueue(ir);
                }
            }
        }

        private List<IRGoal> ParallelBuildIR()
        {
            var tasks = new IRBuildTasks();
            foreach (var script in GoalScripts)
            {
                tasks.Inputs.Enqueue(script);
            }

            IRBuildTasks[] threadTasks = new[] { tasks, tasks, tasks, tasks };
            Task.WhenAll(threadTasks.Select(task => Task.Run(() => { BuildIR(task); }))).Wait();

            var sorted = new SortedDictionary<string, IRGoal>();
            while (tasks.IRs.TryDequeue(out IRGoal goal))
            {
                sorted[goal.Name] = goal;
            }

            return sorted.Values.ToList();
        }

        private void LoadGameObjects(Resource resource)
        {
            if (!resource.Regions.TryGetValue("Templates", out Region templates))
            {
                // TODO - log error
                return;
            }

            if (!templates.Children.TryGetValue("GameObjects", out List<LSLib.LS.Node> gameObjects))
            {
                // TODO - log error
                return;
            }

            foreach (var gameObject in gameObjects)
            {
                if (gameObject.Attributes.TryGetValue("MapKey", out NodeAttribute objectGuid)
                    && gameObject.Attributes.TryGetValue("Name", out NodeAttribute objectName)
                    && gameObject.Attributes.TryGetValue("Type", out NodeAttribute objectType))
                {
                    LSLib.LS.Story.Compiler.ValueType type = null;
                    switch ((string)objectType.Value)
                    {
                        case "item": type = Compiler.Context.LookupType("ITEMGUID"); break;
                        case "character": type = Compiler.Context.LookupType("CHARACTERGUID"); break;
                        case "trigger": type = Compiler.Context.LookupType("TRIGGERGUID"); break;
                        default:
                            // TODO - log unknown type
                            break;
                    }

                    if (type != null)
                    {
                        var gameObjectInfo = new GameObjectInfo
                        {
                            Name = objectName.Value + "_" + objectGuid.Value,
                            Type = type
                        };
                        Compiler.Context.GameObjects[(string)objectGuid.Value] = gameObjectInfo;
                    }
                }
            }
        }

        private void LoadGlobals()
        {
            foreach (var lsf in GameObjectLSFs)
            {
                using (var stream = new MemoryStream(lsf))
                using (var reader = new LSFReader(stream))
                {
                    var resource = reader.Read();
                    LoadGameObjects(resource);
                }
            }
        }

        private void LoadGoals(ModInfo mod)
        {
            foreach (var file in mod.Scripts)
            {
                var scriptStream = file.Value.MakeStream();
                try
                {
                    using (var reader = new BinaryReader(scriptStream))
                    {
                        string path;
                        if (file.Value is PackagedFileInfo)
                        {
                            var pkgd = file.Value as PackagedFileInfo;
                            path = (pkgd.PackageStream as FileStream).Name + ":/" + pkgd.Name;
                        }
                        else
                        {
                            var fs = file.Value as FilesystemFileInfo;
                            path = fs.FilesystemPath;
                        }

                        var script = new GoalScript
                        {
                            Name = Path.GetFileNameWithoutExtension(file.Value.Name),
                            Path = path,
                            ScriptBody = reader.ReadBytes((int)scriptStream.Length)
                        };
                        GoalScripts.Add(script);
                    }
                }
{'fraction_non_alphanumeric': 0.08239018599129402, 'fraction_numerical': 0.0, 'mean_word_length': 1.2878870179250408, 'pattern_counts': {'":': 7, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7192708', 'n_tokens_mistral': 3080, 'n_tokens_neox': 2909, 'n_words': 669}