<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using AxeSoftware.Quest.Scripts;

// AttributeLoaders load properties which are stored in the XML as simple values.
// ExtendedAttributeLoaders load properties which are stored with nested XML.

namespace AxeSoftware.Quest
{
    partial class GameLoader
    {
        private interface IExtendedAttributeLoader
        {
            string AppliesTo { get; }
            void Load(XmlReader reader, Element current);
            GameLoader GameLoader { set; }
            bool SupportsMode(LoadMode mode);
        }

        private Dictionary<string, IExtendedAttributeLoader> m_extendedAttributeLoaders = new Dictionary<string, IExtendedAttributeLoader>();

        private void AddExtendedAttributeLoaders(LoadMode mode)
        {
            foreach (Type t in AxeSoftware.Utility.Classes.GetImplementations(System.Reflection.Assembly.GetExecutingAssembly(),
                typeof(IExtendedAttributeLoader)))
            {
                AddExtendedAttributeLoader((IExtendedAttributeLoader)Activator.CreateInstance(t), mode);
            }
        }

        private void AddExtendedAttributeLoader(IExtendedAttributeLoader loader, LoadMode mode)
        {
            if (loader.SupportsMode(mode))
            {
                m_extendedAttributeLoaders.Add(loader.AppliesTo, loader);
                loader.GameLoader = this;
            }
        }

        private abstract class ExtendedAttributeLoaderBase : IExtendedAttributeLoader
        {
            public abstract string AppliesTo { get; }

            public abstract void Load(XmlReader reader, Element current);

            public GameLoader GameLoader
            {
                get;
                set;
            }

            public virtual bool SupportsMode(LoadMode mode)
            {
                return true;
            }
        }

        private class ScriptDictionaryLoader : ExtendedAttributeLoaderBase
        {
            public override string AppliesTo
            {
                get { return "scriptdictionary"; }
            }

            public override void Load(XmlReader reader, Element current)
            {
                string currentXmlElementName = reader.Name;
                Dictionary<string, string> result = LoadScriptDictionary(reader, current, currentXmlElementName);
                current.Fields.LazyFields.AddScriptDictionary(currentXmlElementName, result);
            }

            private Dictionary<string, string> LoadScriptDictionary(XmlReader reader, Element current, string xmlElementName)
            {
                Dictionary<string, string> newDictionary = new Dictionary<string, string>();
                if (reader.IsEmptyElement) return newDictionary;

                int nestCount = 1;
                string key = null;
                string value = string.Empty;

                while (reader.Read())
                {
                    switch (reader.NodeType)
                    {
                        case XmlNodeType.Element:
                            nestCount++;
                            if (reader.Name == "item")
                            {
                                key = reader.GetAttribute("key");
                                value = string.Empty;
                            }
                            else
                            {
                                throw new InvalidOperationException(string.Format("Invalid element '{0}' in scriptdictionary block for '{1}.{2}' - expected only 'item' elements", reader.Name, current.Name, xmlElementName));
                            }
                            break;
                        case XmlNodeType.EndElement:
                            if (reader.Name == "item")
                            {
                                newDictionary.Add(key, value);
                            }

                            nestCount--;

                            if (nestCount == 0)
                            {
                                // Finished reading this XML block. Sanity check that we are
                                // closing the element that we opened with.
                                if (reader.Name == xmlElementName)
                                {
                                    return newDictionary;
                                }

                                throw new InvalidOperationException(string.Format("Expected closing tag '{0}'", xmlElementName));
                            }
                            break;
                        case XmlNodeType.Text:
                            value = reader.Value;
                            break;
                    }
                }

                throw new Exception("Unexpected end of XML data");
            }<commit_msg>#850 - CDATA sections were ignored when loading script dictionary<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using AxeSoftware.Quest.Scripts;

// AttributeLoaders load properties which are stored in the XML as simple values.
// ExtendedAttributeLoaders load properties which are stored with nested XML.

namespace AxeSoftware.Quest
{
    partial class GameLoader
    {
        private interface IExtendedAttributeLoader
        {
            string AppliesTo { get; }
            void Load(XmlReader reader, Element current);
            GameLoader GameLoader { set; }
            bool SupportsMode(LoadMode mode);
        }

        private Dictionary<string, IExtendedAttributeLoader> m_extendedAttributeLoaders = new Dictionary<string, IExtendedAttributeLoader>();

        private void AddExtendedAttributeLoaders(LoadMode mode)
        {
            foreach (Type t in AxeSoftware.Utility.Classes.GetImplementations(System.Reflection.Assembly.GetExecutingAssembly(),
                typeof(IExtendedAttributeLoader)))
            {
                AddExtendedAttributeLoader((IExtendedAttributeLoader)Activator.CreateInstance(t), mode);
            }
        }

        private void AddExtendedAttributeLoader(IExtendedAttributeLoader loader, LoadMode mode)
        {
            if (loader.SupportsMode(mode))
            {
                m_extendedAttributeLoaders.Add(loader.AppliesTo, loader);
                loader.GameLoader = this;
            }
        }

        private abstract class ExtendedAttributeLoaderBase : IExtendedAttributeLoader
        {
            public abstract string AppliesTo { get; }

            public abstract void Load(XmlReader reader, Element current);

            public GameLoader GameLoader
            {
                get;
                set;
            }

            public virtual bool SupportsMode(LoadMode mode)
            {
                return true;
            }
        }

        private class ScriptDictionaryLoader : ExtendedAttributeLoaderBase
        {
            public override string AppliesTo
            {
                get { return "scriptdictionary"; }
            }

            public override void Load(XmlReader reader, Element current)
            {
                string currentXmlElementName = reader.Name;
                Dictionary<string, string> result = LoadScriptDictionary(reader, current, currentXmlElementName);
                current.Fields.LazyFields.AddScriptDictionary(currentXmlElementName, result);
            }

            private Dictionary<string, string> LoadScriptDictionary(XmlReader reader, Element current, string xmlElementName)
            {
                Dictionary<string, string> newDictionary = new Dictionary<string, string>();
                if (reader.IsEmptyElement) return newDictionary;

                int nestCount = 1;
                string key = null;
                string value = string.Empty;

                while (reader.Read())
                {
                    switch (reader.NodeType)
                    {
                        case XmlNodeType.Element:
                            nestCount++;
                            if (reader.Name == "item")
                            {
                                key = reader.GetAttribute("key");
                                value = string.Empty;
                            }
                            else
                            {
                                throw new InvalidOperationException(string.Format("Invalid element '{0}' in scriptdictionary block for '{1}.{2}' - expected only 'item' elements", reader.Name, current.Name, xmlElementName));
                            }
                            break;
                        case XmlNodeType.EndElement:
                            if (reader.Name == "item")
                            {
                                newDictionary.Add(key, value);
                            }

                            nestCount--;

                            if (nestCount == 0)
                            {
                                // Finished reading this XML block. Sanity check that we are
                                // closing the element that we opened with.
                                if (reader.Name == xmlElementName)
                                {
                                    return newDictionary;
                                }

                                throw new InvalidOperationException(string.Format("Expected closing tag '{0}'", xmlElementName));
                            }
                            break;
                        case XmlNodeType.Text:
                        case XmlNodeType.CDATA:
                            value = reader.Value;
                            break;
                    }
                }

                throw new Exception("Unexpected end of XML data");
            }
{'fraction_non_alphanumeric': 0.06359428456429865, 'fraction_numerical': 0.0015093580197222782, 'mean_word_length': 1.2754120879120878, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5998403', 'n_tokens_mistral': 2136, 'n_tokens_neox': 1977, 'n_words': 526}