<commit_before>﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using UnityEditor;
using UnityEditor.Compilation;
using UnityEditorInternal;
using UnityEngine;
using Object = UnityEngine.Object;

namespace Microsoft.MixedReality.Toolkit.MSBuild
{
    public static class AssetScriptReferenceRetargeter
    {
        private struct ClassInformation
        {
            public string Name;
            public string Namespace;
            public string Guid;
            public long FileId;
            public int ExecutionOrder;
        }

        private class AssemblyInformation
        {
            public string Name { get; }

            public string Guid { get; }

            public Dictionary<string, ClassInformation> CompiledClasses { get; }

            public Dictionary<string, int> ExecutionOrderEntries { get; }

            public AssemblyInformation(string name, string dllGuid)
            {
                Name = name;
                Guid = dllGuid;
                CompiledClasses = new Dictionary<string, ClassInformation>();
                ExecutionOrderEntries = new Dictionary<string, int>();
            }
        }

        private const string YamlPrefix = "%YAML 1.1";

        private static readonly Dictionary<string, string> SourceToOutputFolders = new Dictionary<string, string>
        {
            { "MSBuild/Publish/Player/Android", "AndroidPlayer" },
            { "MSBuild/Publish/Player/iOS", "iOSPlayer" },
            { "MSBuild/Publish/Player/WSA", "UAPPlayer" },
            { "MSBuild/Publish/Player/WindowsStandalone32", "StandalonePlayer" },
        };

        private static readonly HashSet<string> ExcludedYamlAssetExtensions = new HashSet<string> { ".jpg", ".csv", ".meta", ".pfx", ".txt", ".nuspec", ".asmdef", ".yml", ".cs", ".md", ".json", ".ttf", ".png", ".shader", ".wav", ".bin", ".gltf", ".glb", ".fbx", ".pdf", ".cginc", ".rsp", ".xml", ".targets", ".props", ".template", ".csproj", ".sln", ".psd", ".room", ".sentinel", ".npmignore", ".bytes" };
        private static readonly HashSet<string> ExcludedSuffixFromCopy = new HashSet<string>() { ".cs", ".cs.meta", ".asmdef", ".asmdef.meta", ".rsp" };

        private static readonly Dictionary<string, string> NonClassDictionary = new Dictionary<string, string>(); // GUID, FileName

        // This is the known Unity-defined script fileId
        private const string ScriptFileIdConstant = "11500000";

        private const string OculusProfileGUID = "4f726b4cb3605994fac74d508110ec62";
        private const string GUIDDictionaryFileName = "mrtk_guid_remapping_dictionary.txt";

        [Obsolete("Obsolete and removed after deprecation of the NuGet distribution. Use RetargetAssetsToScript() to retarget to script GUIDs.")]
        public static void RetargetAssets()
        {
            try
            {
                Debug.Log("Starting to retarget assets.");
                RunRetargetToDLL();
                Debug.Log("Completed asset retargeting.");
            }
            catch (Exception ex)
            {
                Debug.LogError("Failed to retarget assets.");
                Debug.LogException(ex);

                throw ex;
            }
        }

        [MenuItem("Mixed Reality Toolkit/MSBuild/Retarget assets to scripts")]
        public static void RetargetAssetsToScript()
        {
            try
            {
                Debug.Log("Starting to retarget assets.");
                RunRetargetToScript();
                Debug.Log("Completed asset retargeting.");
            }
            catch (Exception ex)
            {
                Debug.LogError("Failed to retarget assets.");
                Debug.LogException(ex);

                throw ex;
            }
        }

        private static void RunRetargetToDLL()
        {
            string[] allFilesUnderAssets = Directory.GetFiles(Application.dataPath, "*", SearchOption.AllDirectories);

            Dictionary<string, ClassInformation> scriptFilesReferences = ProcessScripts(allFilesUnderAssets);
            Debug.Log($"Found {scriptFilesReferences.Count} script file references.");

            // DLL name to GUID
            Dictionary<string, string> asmDefMappings = RetrieveAsmDefGuids(allFilesUnderAssets);

            Dictionary<string, AssemblyInformation> compiledClassReferences = ProcessCompiledDLLs("PackagedAssemblies", Application.dataPath.Replace("Assets", "NuGet/Plugins/EditorPlayer"), asmDefMappings);
            Debug.Log($"Found {compiledClassReferences.Select(t => t.Value.CompiledClasses.Count).Sum()} compiled class references.");

            Dictionary<string, Tuple<string, long>> remapDictionary = new Dictionary<string, Tuple<string, long>>();

            foreach (KeyValuePair<string, AssemblyInformation> pair in compiledClassReferences)
            {
                foreach (KeyValuePair<string, ClassInformation> compiledClass in pair.Value.CompiledClasses)
                {
                    ClassInformation compiledClassInfo = compiledClass.Value;
                    if (scriptFilesReferences.TryGetValue(compiledClass.Key, out ClassInformation scriptClassInfo))
                    {
                        if (scriptClassInfo.ExecutionOrder != 0)
                        {
                            pair.Value.ExecutionOrderEntries.Add($"{scriptClassInfo.Namespace}.{scriptClassInfo.Name}", scriptClassInfo.ExecutionOrder);
                        }

                        remapDictionary.Add(scriptClassInfo.Guid, new Tuple<string, long>(compiledClassInfo.Guid, compiledClassInfo.FileId));
                        scriptFilesReferences.Remove(compiledClass.Key);
                    }
                    else
                    {
                        Debug.LogWarning($"Can't find a script version of the compiled class: {compiledClass.Key}; {pair.Key}.dll. This generally means the compiled class is second or later in a script file, and Unity doesn't parse it as two different assets.");
                    }
                }
            }

            ProcessYAMLAssets(allFilesUnderAssets, Application.dataPath.Replace("Assets", "NuGet/Content"), remapDictionary, compiledClassReferences);

            string folderPath = null;
            string[] arguments = Environment.GetCommandLineArgs();

            for (int i = 0; i < arguments.Length; ++i)
            {
                switch (arguments[i])
                {
                    case "-dictionaryFileOutputFolder":
                        folderPath = arguments[++i];
                        break;

                }
            }

            if (!string.IsNullOrWhiteSpace(folderPath))
            {
                string filePath = Path.Combine(folderPath, GUIDDictionaryFileName);
                Debug.Log($"Writing remapping dictionary to {filePath}");
                File.WriteAllLines(filePath, remapDictionary.Select(x => $"{x.Value.Item2} | {x.Key}, {ScriptFileIdConstant}"));
                if (filePath.Contains("Assets"))
                {
                    string nugetFilePath = filePath.Replace("Assets", "NuGet/Content");
                    Debug.Log($"Copying remapping dictionary to {nugetFilePath}");
                    File.Copy(filePath, nugetFilePath, true);
                }
            }
        }

        private static void RunRetargetToScript()
        {
            string[] dictionaryPaths = AssetDatabase.FindAssets(Path.GetFileNameWithoutExtension(GUIDDictionaryFileName));
            if (dictionaryPaths.Length != 1)
            {
                Debug.LogError($"Couldn't locate single mapping file. Found {dictionaryPaths.Length}.");
                return;
            }

            Dictionary<string, Tuple<string, long>> remapDictionary = ReadDictionaryFile(File.ReadLines(Path.GetFullPath(AssetDatabase.GUIDToAssetPath(dictionaryPaths[0]))));

            ProcessYAMLAssets(remapDictionary);
        }

        private static Dictionary<string, Tuple<string, long>> ReadDictionaryFile(IEnumerable<string> dictionaryFileLines)
        {
            Dictionary<string, Tuple<string, long>> returnDictionary = new Dictionary<string, Tuple<string, long>>();

            foreach (string line in dictionaryFileLines)
            {
                string[] split = line.Split('|');
                string[] split2 = split[1].Split(',');
                returnDictionary.Add(split[0].Trim(), new Tuple<string, long>(split2[0].Trim(), long.Parse(split2[1].Trim())));
            }

            return returnDictionary;
        }

        private static Dictionary<string, string> RetrieveAsmDefGuids(string[] allFiles)
        {
            int lengthOfPrefix = Application.dataPath.IndexOf("Assets");
            Dictionary<string, string> dllGuids = new Dictionary<string, string>();
            foreach (string asmdefFile in allFiles.Where(t => t.EndsWith(".asmdef")))
            {
                string asmdefText = AssetDatabase.LoadAssetAtPath<AssemblyDefinitionAsset>(asmdefFile.Substring(lengthOfPrefix)).text;
                string dllName = JsonUtility.FromJson<AssemblyDefinitionStub>(asmdefText).name;
                string guid = File.ReadAllLines($"{asmdefFile}.meta")[1].Substring(6);
                if (!Guid.TryParse(guid, out Guid _))
                {
                    throw new InvalidDataException("AsmDef meta file must have changed, as we can no longer parse a GUID out of it.");
                }
                guid = CycleGuidForward(guid);
                dllGuids.Add($"{dllName}.dll", guid);
            }

            return dllGuids;
        }

        private static void ProcessYAMLAssets(Dictionary<string, Tuple<string, long>> remapDictionary)
        {
            HashSet<string> yamlAssets = new HashSet<string>();
            string[] allFilePaths = Directory.GetFiles(Application.dataPath, "*", SearchOption.AllDirectories);<commit_msg>Update dictionary reading to be more robust to malformed txt files<commit_after>﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using UnityEditor;
using UnityEditor.Compilation;
using UnityEditorInternal;
using UnityEngine;
using Object = UnityEngine.Object;

namespace Microsoft.MixedReality.Toolkit.MSBuild
{
    public static class AssetScriptReferenceRetargeter
    {
        private struct ClassInformation
        {
            public string Name;
            public string Namespace;
            public string Guid;
            public long FileId;
            public int ExecutionOrder;
        }

        private class AssemblyInformation
        {
            public string Name { get; }

            public string Guid { get; }

            public Dictionary<string, ClassInformation> CompiledClasses { get; }

            public Dictionary<string, int> ExecutionOrderEntries { get; }

            public AssemblyInformation(string name, string dllGuid)
            {
                Name = name;
                Guid = dllGuid;
                CompiledClasses = new Dictionary<string, ClassInformation>();
                ExecutionOrderEntries = new Dictionary<string, int>();
            }
        }

        private const string YamlPrefix = "%YAML 1.1";

        private static readonly Dictionary<string, string> SourceToOutputFolders = new Dictionary<string, string>
        {
            { "MSBuild/Publish/Player/Android", "AndroidPlayer" },
            { "MSBuild/Publish/Player/iOS", "iOSPlayer" },
            { "MSBuild/Publish/Player/WSA", "UAPPlayer" },
            { "MSBuild/Publish/Player/WindowsStandalone32", "StandalonePlayer" },
        };

        private static readonly HashSet<string> ExcludedYamlAssetExtensions = new HashSet<string> { ".jpg", ".csv", ".meta", ".pfx", ".txt", ".nuspec", ".asmdef", ".yml", ".cs", ".md", ".json", ".ttf", ".png", ".shader", ".wav", ".bin", ".gltf", ".glb", ".fbx", ".pdf", ".cginc", ".rsp", ".xml", ".targets", ".props", ".template", ".csproj", ".sln", ".psd", ".room", ".sentinel", ".npmignore", ".bytes" };
        private static readonly HashSet<string> ExcludedSuffixFromCopy = new HashSet<string>() { ".cs", ".cs.meta", ".asmdef", ".asmdef.meta", ".rsp" };

        private static readonly Dictionary<string, string> NonClassDictionary = new Dictionary<string, string>(); // GUID, FileName

        // This is the known Unity-defined script fileId
        private const string ScriptFileIdConstant = "11500000";

        private const string OculusProfileGUID = "4f726b4cb3605994fac74d508110ec62";
        private const string GUIDDictionaryFileName = "mrtk_guid_remapping_dictionary.txt";

        [Obsolete("Obsolete and removed after deprecation of the NuGet distribution. Use RetargetAssetsToScript() to retarget to script GUIDs.")]
        public static void RetargetAssets()
        {
            try
            {
                Debug.Log("Starting to retarget assets.");
                RunRetargetToDLL();
                Debug.Log("Completed asset retargeting.");
            }
            catch (Exception ex)
            {
                Debug.LogError("Failed to retarget assets.");
                Debug.LogException(ex);

                throw ex;
            }
        }

        [MenuItem("Mixed Reality Toolkit/MSBuild/Retarget assets to scripts")]
        public static void RetargetAssetsToScript()
        {
            try
            {
                Debug.Log("Starting to retarget assets.");
                RunRetargetToScript();
                Debug.Log("Completed asset retargeting.");
            }
            catch (Exception ex)
            {
                Debug.LogError("Failed to retarget assets.");
                Debug.LogException(ex);

                throw ex;
            }
        }

        private static void RunRetargetToDLL()
        {
            string[] allFilesUnderAssets = Directory.GetFiles(Application.dataPath, "*", SearchOption.AllDirectories);

            Dictionary<string, ClassInformation> scriptFilesReferences = ProcessScripts(allFilesUnderAssets);
            Debug.Log($"Found {scriptFilesReferences.Count} script file references.");

            // DLL name to GUID
            Dictionary<string, string> asmDefMappings = RetrieveAsmDefGuids(allFilesUnderAssets);

            Dictionary<string, AssemblyInformation> compiledClassReferences = ProcessCompiledDLLs("PackagedAssemblies", Application.dataPath.Replace("Assets", "NuGet/Plugins/EditorPlayer"), asmDefMappings);
            Debug.Log($"Found {compiledClassReferences.Select(t => t.Value.CompiledClasses.Count).Sum()} compiled class references.");

            Dictionary<string, Tuple<string, long>> remapDictionary = new Dictionary<string, Tuple<string, long>>();

            foreach (KeyValuePair<string, AssemblyInformation> pair in compiledClassReferences)
            {
                foreach (KeyValuePair<string, ClassInformation> compiledClass in pair.Value.CompiledClasses)
                {
                    ClassInformation compiledClassInfo = compiledClass.Value;
                    if (scriptFilesReferences.TryGetValue(compiledClass.Key, out ClassInformation scriptClassInfo))
                    {
                        if (scriptClassInfo.ExecutionOrder != 0)
                        {
                            pair.Value.ExecutionOrderEntries.Add($"{scriptClassInfo.Namespace}.{scriptClassInfo.Name}", scriptClassInfo.ExecutionOrder);
                        }

                        remapDictionary.Add(scriptClassInfo.Guid, new Tuple<string, long>(compiledClassInfo.Guid, compiledClassInfo.FileId));
                        scriptFilesReferences.Remove(compiledClass.Key);
                    }
                    else
                    {
                        Debug.LogWarning($"Can't find a script version of the compiled class: {compiledClass.Key}; {pair.Key}.dll. This generally means the compiled class is second or later in a script file, and Unity doesn't parse it as two different assets.");
                    }
                }
            }

            ProcessYAMLAssets(allFilesUnderAssets, Application.dataPath.Replace("Assets", "NuGet/Content"), remapDictionary, compiledClassReferences);

            string folderPath = null;
            string[] arguments = Environment.GetCommandLineArgs();

            for (int i = 0; i < arguments.Length; ++i)
            {
                switch (arguments[i])
                {
                    case "-dictionaryFileOutputFolder":
                        folderPath = arguments[++i];
                        break;

                }
            }

            if (!string.IsNullOrWhiteSpace(folderPath))
            {
                string filePath = Path.Combine(folderPath, GUIDDictionaryFileName);
                Debug.Log($"Writing remapping dictionary to {filePath}");
                File.WriteAllLines(filePath, remapDictionary.Select(x => $"{x.Value.Item2} | {x.Key}, {ScriptFileIdConstant}"));
                if (filePath.Contains("Assets"))
                {
                    string nugetFilePath = filePath.Replace("Assets", "NuGet/Content");
                    Debug.Log($"Copying remapping dictionary to {nugetFilePath}");
                    File.Copy(filePath, nugetFilePath, true);
                }
            }
        }

        private static void RunRetargetToScript()
        {
            string[] dictionaryPaths = AssetDatabase.FindAssets(Path.GetFileNameWithoutExtension(GUIDDictionaryFileName));
            if (dictionaryPaths.Length != 1)
            {
                Debug.LogError($"Couldn't locate single mapping file. Found {dictionaryPaths.Length}.");
                return;
            }

            Dictionary<string, Tuple<string, long>> remapDictionary = ReadDictionaryFile(File.ReadLines(Path.GetFullPath(AssetDatabase.GUIDToAssetPath(dictionaryPaths[0]))));

            if (remapDictionary.Count > 0)
            {
                ProcessYAMLAssets(remapDictionary);
            }
            else
            {
                Debug.LogError("No valid dictionary mapping file was found.");
            }
        }

        private static Dictionary<string, Tuple<string, long>> ReadDictionaryFile(IEnumerable<string> dictionaryFileLines)
        {
            Dictionary<string, Tuple<string, long>> returnDictionary = new Dictionary<string, Tuple<string, long>>();

            foreach (string line in dictionaryFileLines)
            {
                string[] split = line.Split('|');
                if (split.Length == 2)
                {
                    string[] split2 = split[1].Split(',');
                    if (split2.Length == 2)
                    {
                        returnDictionary.Add(split[0].Trim(), new Tuple<string, long>(split2[0].Trim(), long.Parse(split2[1].Trim())));
                    }
                }
            }

            return returnDictionary;
        }

        private static Dictionary<string, string> RetrieveAsmDefGuids(string[] allFiles)
        {
            int lengthOfPrefix = Application.dataPath.IndexOf("Assets");
            Dictionary<string, string> dllGuids = new Dictionary<string, string>();
            foreach (string asmdefFile in allFiles.Where(t => t.EndsWith(".asmdef")))
            {
                string asmdefText = AssetDatabase.LoadAssetAtPath<AssemblyDefinitionAsset>(asmdefFile.Substring(lengthOfPrefix)).text;
                string dllName = JsonUtility.FromJson<AssemblyDefinitionStub>(asmdefText).name;
                string guid = File.ReadAllLines($"{asmdefFile}.meta")[1].Substring(6);
                if (!Guid.TryParse(guid, out Guid _))
                {
                    throw new InvalidDataException("AsmDef meta file must have changed, as we can no longer parse a GUID out of it.");
                }
                guid = CycleGuidForward(guid);
                dllGuids.Add($"{dllName}.dll", guid);
            }

            return dllGuids;
        }

        private static void ProcessYAMLAssets(Dictionary<string, Tuple<string, long>> remapDictionary)
        {
            HashSet<string> yamlAssets = new HashSet<string>();
            string[] allFilePaths = Directory.GetFiles(Application.dataPath, "*", SearchOption.AllDirectories);
{'fraction_non_alphanumeric': 0.10460657249648075, 'fraction_numerical': 0.004854133294500267, 'mean_word_length': 2.119624470018171, 'pattern_counts': {'":': 2, '<': 87, '<?xml version=': 0, '>': 92, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5325765', 'n_tokens_mistral': 5310, 'n_tokens_neox': 4962, 'n_words': 1168}