<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Web.Helpers;
using System.Xml;
using System.Xml.Linq;
using Sdl.Web.Templating.ExtensionMethods;
using Tridion.ContentManager;
using Tridion.ContentManager.CommunicationManagement;
using Tridion.ContentManager.ContentManagement;
using Tridion.ContentManager.ContentManagement.Fields;
using Tridion.ContentManager.Templating;
using Tridion.ContentManager.Templating.Assembly;

namespace Sdl.Web.Templating
{
    /// <summary>
    /// Publish all files required to make a version release for the implementation:
    /// * Configuration
    /// * Resources
    /// * Schema, Template and Taxonomy ids (these are published only from the default localization/master web publication)
    /// * Design Elements (if applicable)
    /// * Views (if applicable)
    /// </summary>
    [TcmTemplateTitle("Publish Version")]
    [TcmTemplateParameterSchema("resource:Sdl.Web.Templating.Resources.Schemas.Publish Version Parameters.xsd")]
    public class PublishVersion : TemplateBase.TemplateBase
    {
        private const string BootstrapFilename = "_all";
        private const string TemplateConfigName = "templates";
        private const string SchemasConfigName = "schemas";
        private const string TaxonomiesConfigName = "taxonomies";
        private const string MappingsConfigName = "mappings";
        private const string VocabulariesConfigName = "vocabularies";
        private const string VocabulariesAppDataId = "http://www.sdl.com/tridion/SemanticMapping/vocabularies";
        private const string TypeOfAppDataId = "http://www.sdl.com/tridion/SemanticMapping/typeof";

        private readonly Dictionary<string, string> _namespaces = new Dictionary<string, string>
            {
                {"xsd", "http://www.w3.org/2001/XMLSchema"},
                {"tcm", "http://www.tridion.com/ContentManager/5.0"},
                {"xlink", "http://www.w3.org/1999/xlink"},
                {"tcmi","http://www.tridion.com/ContentManager/5.0/Instance"},
                {"mapping", "http://www.sdl.com/tridion/SemanticMapping"}
            };

        private string _moduleRoot = string.Empty;

        public override void Transform(Engine engine, Package package)
        {
            Initialize(engine, package);
            var folderWebDavUrl = GetPublication().RootFolder.WebDavUrl + "/" + package.GetValue("folderWebDavUrl");
            var sgUri = MPackage.GetValue("sg_TargetStructureGroup");
            var sg = (StructureGroup)MEngine.GetObject(engine.LocalizeUri(new TcmUri(sgUri)));
            Folder folder;
            try
            {
                folder = (Folder)MEngine.GetObject(folderWebDavUrl);
            }
            catch
            {
                throw new Exception(String.Format("Cannot find modules folder: {0}. Please check it exists and update your TBB parameters accordingly.", folderWebDavUrl));
            }
            _moduleRoot = folder.WebDavUrl;
            if (sg == null)
            {
                throw new Exception(String.Format("Could not find structure group {0} in context publication {1}", sgUri, GetPublication().Title));
            }
            List<string> bootstrapFiles = new List<string>
                {
                    Json.Encode(PublishAssets(folder, sg, "config")),
                    Json.Encode(PublishAssets(folder, sg, "mappings")),
                    Json.Encode(PublishAssets(folder, sg, "resources"))
                };
            Item jsonBootstrap = MPackage.CreateStringItem(ContentType.Text, String.Format("{{\"files\":[{0}]}}", String.Join(",", bootstrapFiles)));
            MPackage.PushItem(Package.OutputName, jsonBootstrap);
            MEngine.PublishingContext.RenderedItem.AddBinary(jsonBootstrap.GetAsStream(), BootstrapFilename + JsonExtension, sg, "bootstrap", GetComponent(), JsonMimetype);
        }

        private string PublishAssets(Folder rootFolder, StructureGroup rootSg, string type)
        {
            Folder folder = (Folder)GetChildOrganizationalItem(rootFolder, "_" + type);
            StructureGroup sg = (StructureGroup)GetChildOrganizationalItem(rootSg, type);
            if (folder == null)
            {
                throw new Exception(String.Format("Could not find Folder '_{0}' in module folder {1}", type, rootFolder.WebDavUrl));
            }
            if (sg == null)
            {
                throw new Exception(String.Format("Could not find Structure Group '{0}' in system Structure Group {1}", type, sg.WebDavUrl));
            }
            List<string> files = new List<string>();
            Component allComponent = null;
            bool isArray = false;
            foreach (var item in GetOrganizationalItemContents(folder, ItemType.Component, false))
            {
                var comp = (Component)MEngine.GetObject(item.Key.ToString());
                var configType = item.Value.ToLower();
                if (configType == BootstrapFilename)
                {
                    //There is a dummy _All component used to publish a bootstrap file containing a list of other files that have been generated
                    allComponent = comp;
                    continue;
                }
                var settings = new Dictionary<string, List<string>>();
                if (configType.Equals(TemplateConfigName) || configType.Equals(SchemasConfigName) || configType.Equals(TaxonomiesConfigName) || configType.Equals(MappingsConfigName))
                {
                    //template, schema and taxonomy config is only published from the master web publication/default localization
                    if (IsMasterWebPublication())
                    {
                        switch (configType)
                        {
                            case TemplateConfigName:
                                isArray = false;
                                settings = ReadTemplateData();
                                break;
                            case SchemasConfigName:
                                isArray = false;
                                settings = ReadSchemaData();
                                break;
                            case TaxonomiesConfigName:
                                isArray = false;
                                settings = ReadTaxonomiesData();
                                break;
                            case MappingsConfigName:
                                isArray = true;
                                settings = ReadMappingsData();
                                break;
                        }
                    }
                }
                else
                {
                    settings = new Dictionary<string, List<string>> { { configType, ReadComponentData(comp) } };
                }
                //Add all the JSON files to the package and publish them as binaries
                foreach (var key in settings.Keys)
                {
                    Item jsonItem;
                    if (isArray)
                    {
                        jsonItem = MPackage.CreateStringItem(ContentType.Text, String.Format("[{0}]", String.Join(",\n", settings[key])));
                    }
                    else
                    {
                        jsonItem = MPackage.CreateStringItem(ContentType.Text, String.Format("{{{0}}}", String.Join(",\n", settings[key])));
                    }
                    var filename = key + JsonExtension;
                    MPackage.PushItem(type + "/" + filename, jsonItem);
                    var binary = MEngine.PublishingContext.RenderedItem.AddBinary(jsonItem.GetAsStream(), filename, sg, key, comp, JsonMimetype);
                    files.Add(Json.Encode(binary.Url));
                }
            }
            if (allComponent == null)
            {
                throw new Exception(String.Format("Cannot find '_All' component. Please ensure that this component is in the {0} folder.", folder.WebDavUrl));
            }
            //We create a special asset file _all.js which contains the path of all other asset files of this type that have been published
            Item filesItem = MPackage.CreateStringItem(ContentType.Text, String.Format("{{\"defaultLocalization\":{0},\"files\":[{1}]}}", Json.Encode(IsMasterWebPublication()), String.Join(",", files)));
            MPackage.PushItem(type + "/" + BootstrapFilename + JsonExtension, filesItem);
            var bootstrapBinary = MEngine.PublishingContext.RenderedItem.AddBinary(filesItem.GetAsStream(), BootstrapFilename + JsonExtension, sg, type, allComponent, JsonMimetype);
            return bootstrapBinary.Url;
        }

        private static List<string> ReadComponentData(Component comp)
        {
            var fields = new ItemFields(comp.Content, comp.Schema);
            var settings = new List<string>();
            var configFields = fields.GetEmbeddedFields("settings");
            if (configFields.Any())
            {
                //either schema is a generic multival embedded name/value
                foreach (var setting in configFields)
                {
                    settings.Add(String.Format("{0}:{1}", Json.Encode(setting.GetTextValue("name")), Json.Encode(setting.GetTextValue("value"))));
                }
            }
            else
            {
                //... or its a custom schema with individual fields
                foreach (var field in fields)
                {
                    //TODO handle more types
                    if (field is TextField)
                    {
                        settings.Add(String.Format("{0}:{1}", Json.Encode(field.Name), Json.Encode(fields.GetTextValue(field.Name))));
                    }
                }
            }
            return settings;
        }

        private Dictionary<string, List<string>> ReadMappingsData()
        {
            // generate a list of vocabulary prefix and name from appdata
            var res = new Dictionary<string, List<string>> { { VocabulariesConfigName, new List<string>() } };
            ApplicationData globalAppData = MEngine.GetSession().SystemManager.LoadGlobalApplicationData(VocabulariesAppDataId);
            if (globalAppData != null)
            {
                XElement vocabulariesXml = XElement.Parse(Encoding.Unicode.GetString(globalAppData.Data));
                foreach (var vocabulary in vocabulariesXml.Elements())
                {
                    res[VocabulariesConfigName].Add(String.Format("{{{0}:{1},{2}:{3}}}", Json.Encode("Prefix"), Json.Encode(vocabulary.Attribute("prefix").Value), Json.Encode("Vocab"), Json.Encode(vocabulary.Attribute("name").Value)));
                }
            }

            // generate a list of schema + id, separated by module
            var schemaFilter = new RepositoryItemsFilter(MEngine.GetSession())
            {
                Recursive = true,
                ItemTypes = new List<ItemType> { ItemType.Schema },
                BaseColumns = ListBaseColumns.Extended
            };
            foreach (XmlElement item in GetPublication().GetListItems(schemaFilter).ChildNodes)
            {
                var type = item.GetAttribute("Type");
                var subType = item.GetAttribute("SubType");
                // we only consider normal schemas (type=8 subtype=0)
                if ((type == "8" && subType == "0"))
                {
                    var id = item.GetAttribute("ID");
                    var schema = (Schema)MEngine.GetObject(id);
                    var module = GetModuleName(schema.WebDavUrl);
                    if (module != null)
                    {
                        var key = module + "." + SchemasConfigName;
                        if (!res.ContainsKey(key))
                        {
                            res.Add(key, new List<string>());
                        }

                        // add schema typeof from appdata
                        StringBuilder schemaSemantics = new StringBuilder();
                        ApplicationData appData = schema.LoadApplicationData(TypeOfAppDataId);
                        if (appData != null)
                        {
                            string typeOf = Encoding.Unicode.GetString(appData.Data);
                            schemaSemantics.Append(BuildSemanticsJson(typeOf, "Vocab", "Entity"));
                        }

                        // TODO: serialize schema fields xml to json in a smart way
                        // field: {"name":"something","isMultiValue":true,"semantics":[],"fields":[]}
                        // field semantics: {"vocab":"s","property":"headline"}
                        StringBuilder fields = new StringBuilder();

                        // load namespace manager with schema namespaces
                        XmlNamespaceManager nsmgr = SchemaNamespaceManager(schema.Xsd.OwnerDocument.NameTable);

                        // build field elements from schema
                        fields.Append(BuildSchemaFieldsJson(schema, nsmgr));

                        res[key].Add(string.Format("{{\"Id\":{0},\"RootElement\":{1},\"Fields\":[{2}],\"Semantics\":[{3}]}}", Json.Encode(schema.Id.ItemId), Json.Encode(schema.RootElementName), fields, schemaSemantics));
                    }
                }
            }
            return res;
        }

        private XmlNamespaceManager SchemaNamespaceManager(XmlNameTable nameTable)
        {
            // load namespace manager with schema namespaces
            XmlNamespaceManager nsmgr = new XmlNamespaceManager(nameTable);
            foreach (var item in _namespaces)
            {
                nsmgr.AddNamespace(item.Key, item.Value);
            }
            return nsmgr;
        }

        // field: {"name":"something","isMultiValue":true,"semantics":[],"fields":[]}
        // field semantics: {"vocab":"s","property":"headline"}
        private string BuildSchemaFieldsJson(Schema schema, XmlNamespaceManager nsmgr, bool embedded = false)
        {
            StringBuilder fields = new StringBuilder();

            // loop over all field elements in schema
            bool first = true;
            string xpath = string.Format("/xsd:schema/xsd:element[@name='{0}']/xsd:complexType/xsd:sequence/xsd:element", schema.RootElementName);
            if (embedded)
            {
                xpath = string.Format("/xsd:schema/xsd:complexType[@name='{0}']/xsd:sequence/xsd:element", schema.RootElementName);
            }
            foreach (XmlNode fieldNode in schema.Xsd.SelectNodes(xpath, nsmgr))
            {
                if (first)
                {
                    first = false;
                }
                else
                {
                    fields.Append(",");
                }
                StringBuilder fieldSemantics = new StringBuilder();

                // if maxOccurs is anything else than 1, it is a multi value field
                bool isMultiValue = !fieldNode.Attributes["maxOccurs"].Value.Equals("1");

                // read semantic mapping from field
                // schema semantics: {"vocab":"s","entity":"Article"}
                XmlNode propertyNode = fieldNode.SelectSingleNode("xsd:annotation/xsd:appinfo/tcm:ExtensionXml/mapping:property", nsmgr);
                string property = propertyNode != null ? propertyNode.InnerText : null;
                fieldSemantics.Append(BuildSemanticsJson(property, "Vocab", "Property"));

                XmlNode typeOfNode = fieldNode.SelectSingleNode("xsd:annotation/xsd:appinfo/tcm:ExtensionXml/mapping:typeof", nsmgr);
                string typeOf = typeOfNode != null ? typeOfNode.InnerText : null;
                if (!string.IsNullOrEmpty(typeOf))
                {
                    if (fieldSemantics.Length > 0)
                    {
                        fieldSemantics.Append(",");
                    }
                    fieldSemantics.Append(BuildSemanticsJson(typeOf, "Vocab", "Entity"));
                }

                // add schema typeof from appdata for embedded schemas
                if (embedded)
                {
                    ApplicationData appData = schema.LoadApplicationData(TypeOfAppDataId);
                    if (appData != null)
                    {
                        if (fieldSemantics.Length > 0)
                        {
                            fieldSemantics.Append(",");
                        }
                        string schemaTypeOf = Encoding.Unicode.GetString(appData.Data);
                        fieldSemantics.Append(BuildSemanticsJson(schemaTypeOf, "Vocab", "Entity"));
                    }
                }

                // handle embedded fields
                StringBuilder embeddedFields = new StringBuilder();
                XmlNode embeddedSchemaNode = fieldNode.SelectSingleNode("xsd:annotation/xsd:appinfo/tcm:EmbeddedSchema", nsmgr);
                if (embeddedSchemaNode != null)
                {
                    string uri = embeddedSchemaNode.Attributes["href", "http://www.w3.org/1999/xlink"].Value;
                    Schema embeddedSchema = (Schema)MEngine.GetObject(uri);
                    embeddedFields.Append(BuildSchemaFieldsJson(embeddedSchema, nsmgr, true));
                }

                // TODO: handle link fields
                //XmlAttribute typeAttribute = fieldNode.Attributes["type"];
                //if (typeAttribute != null)
                //{
                //    bool isSimpleLink = typeAttribute.Value.Equals("tcmi:SimpleLink");
                //    XmlNode allowedTargetSchemasNode = fieldNode.SelectSingleNode("xsd:annotation/xsd:appinfo/tcm:AllowedTargetSchemas", nsmgr);
                //    if (allowedTargetSchemasNode != null)
                //    {
                //        foreach (XmlNode allowedTargetSchemaNode in allowedTargetSchemasNode.SelectNodes("tcm:TargetSchema", nsmgr))
                //        {
                //            string uri = allowedTargetSchemaNode.Attributes["href", "http://www.w3.org/1999/xlink"].Value;
                //            Schema allowedTargetSchema = (Schema)MEngine.GetObject(uri);

                //        }
                //    }
                //    else
                //    {
                //        // if there are no allowed target schemas, all schemas are allowed...
                //    }
                //}
                //else
                //{
                //    // if there is no type attribute, it is not a simple type so look for <xsd:complexType> inside the element
                //}

                fields.AppendFormat("{{\"Name\":{0},\"IsMultiValue\":{1},\"Semantics\":[{2}],\"Fields\":[{3}]}}",
                    Json.Encode(fieldNode.Attributes["name"].Value),
                    Json.Encode(isMultiValue), fieldSemantics, embeddedFields);
            }

            return fields.ToString();
        }

        // schema semantics: {"Vocab":"s","Entity":"Article"}
        // field semantics: {"Vocab":"s","Property":"headline"}
        // semantics: {"prefixName":"a","itemName":"b"},{"prefixName":"c","itemName":"d"}
        private static string BuildSemanticsJson(string input, string prefixName, string itemName)
        {
            StringBuilder semantics = new StringBuilder();
            if (!string.IsNullOrEmpty(input))
            {
                // input = "s:Article" but can also be "s:Article,x:Something"
                string[] values = input.Split(',');
                bool first = true;
                foreach (var value in values)
                {
                    if (first)
                    {
                        first = false;
                    }
                    else
                    {
                        semantics.Append(",");
                    }
                    string[] parts = value.Split(':');
                    semantics.AppendFormat("{{{0}:{1},{2}:{3}}}", Json.Encode(prefixName), Json.Encode(parts[0]), Json.Encode(itemName), Json.Encode(parts[1]));
                }
            }

            return semantics.ToString();
        }

        private Dictionary<string, List<string>> ReadTaxonomiesData()
        {
            //Generate a list of taxonomy + id
            var res = new Dictionary<string, List<string>>();
            var settings = new List<string>();
            var taxFilter = new TaxonomiesFilter(MEngine.GetSession()) { BaseColumns = ListBaseColumns.Extended };
            foreach (XmlElement item in GetPublication().GetListTaxonomies(taxFilter).ChildNodes)
            {
                var id = item.GetAttribute("ID");
                var taxonomy = (Category)MEngine.GetObject(id);
                settings.Add(String.Format("{0}:{1}", Json.Encode(GetKeyFromTaxonomy(taxonomy)), Json.Encode(taxonomy.Id.ItemId)));
            }
            res.Add("core." + TaxonomiesConfigName, settings);
            return res;
        }

        private Dictionary<string, List<string>> ReadSchemaData()<commit_msg>changed semantic mapping json format to contain prefix, entity name and property name for field semantics.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Web.Helpers;
using System.Xml;
using System.Xml.Linq;
using Sdl.Web.Templating.ExtensionMethods;
using Tridion.ContentManager;
using Tridion.ContentManager.CommunicationManagement;
using Tridion.ContentManager.ContentManagement;
using Tridion.ContentManager.ContentManagement.Fields;
using Tridion.ContentManager.Templating;
using Tridion.ContentManager.Templating.Assembly;

namespace Sdl.Web.Templating
{
    /// <summary>
    /// Publish all files required to make a version release for the implementation:
    /// * Configuration
    /// * Resources
    /// * Schema, Template and Taxonomy ids (these are published only from the default localization/master web publication)
    /// * Design Elements (if applicable)
    /// * Views (if applicable)
    /// </summary>
    [TcmTemplateTitle("Publish Version")]
    [TcmTemplateParameterSchema("resource:Sdl.Web.Templating.Resources.Schemas.Publish Version Parameters.xsd")]
    public class PublishVersion : TemplateBase.TemplateBase
    {
        private const string BootstrapFilename = "_all";
        private const string TemplateConfigName = "templates";
        private const string SchemasConfigName = "schemas";
        private const string TaxonomiesConfigName = "taxonomies";
        private const string MappingsConfigName = "mappings";
        private const string VocabulariesConfigName = "vocabularies";
        private const string VocabulariesAppDataId = "http://www.sdl.com/tridion/SemanticMapping/vocabularies";
        private const string TypeOfAppDataId = "http://www.sdl.com/tridion/SemanticMapping/typeof";
        private const string TsiPrefix = "tsi";
        private const string TsiVocabulary = "http://www.sdl.com/tridion/schemas/core";

        private readonly Dictionary<string, string> _namespaces = new Dictionary<string, string>
            {
                {"xsd", "http://www.w3.org/2001/XMLSchema"},
                {"tcm", "http://www.tridion.com/ContentManager/5.0"},
                {"xlink", "http://www.w3.org/1999/xlink"},
                {"tcmi","http://www.tridion.com/ContentManager/5.0/Instance"},
                {"mapping", "http://www.sdl.com/tridion/SemanticMapping"}
            };

        private string _moduleRoot = string.Empty;

        public override void Transform(Engine engine, Package package)
        {
            Initialize(engine, package);
            var folderWebDavUrl = GetPublication().RootFolder.WebDavUrl + "/" + package.GetValue("folderWebDavUrl");
            var sgUri = MPackage.GetValue("sg_TargetStructureGroup");
            var sg = (StructureGroup)MEngine.GetObject(engine.LocalizeUri(new TcmUri(sgUri)));
            Folder folder;
            try
            {
                folder = (Folder)MEngine.GetObject(folderWebDavUrl);
            }
            catch
            {
                throw new Exception(String.Format("Cannot find modules folder: {0}. Please check it exists and update your TBB parameters accordingly.", folderWebDavUrl));
            }
            _moduleRoot = folder.WebDavUrl;
            if (sg == null)
            {
                throw new Exception(String.Format("Could not find structure group {0} in context publication {1}", sgUri, GetPublication().Title));
            }
            List<string> bootstrapFiles = new List<string>
                {
                    Json.Encode(PublishAssets(folder, sg, "config")),
                    Json.Encode(PublishAssets(folder, sg, "mappings")),
                    Json.Encode(PublishAssets(folder, sg, "resources"))
                };
            Item jsonBootstrap = MPackage.CreateStringItem(ContentType.Text, String.Format("{{\"files\":[{0}]}}", String.Join(",", bootstrapFiles)));
            MPackage.PushItem(Package.OutputName, jsonBootstrap);
            MEngine.PublishingContext.RenderedItem.AddBinary(jsonBootstrap.GetAsStream(), BootstrapFilename + JsonExtension, sg, "bootstrap", GetComponent(), JsonMimetype);
        }

        private string PublishAssets(Folder rootFolder, StructureGroup rootSg, string type)
        {
            Folder folder = (Folder)GetChildOrganizationalItem(rootFolder, "_" + type);
            StructureGroup sg = (StructureGroup)GetChildOrganizationalItem(rootSg, type);
            if (folder == null)
            {
                throw new Exception(String.Format("Could not find Folder '_{0}' in module folder {1}", type, rootFolder.WebDavUrl));
            }
            if (sg == null)
            {
                throw new Exception(String.Format("Could not find Structure Group '{0}' in system Structure Group {1}", type, sg.WebDavUrl));
            }
            List<string> files = new List<string>();
            Component allComponent = null;
            bool isArray = false;
            foreach (var item in GetOrganizationalItemContents(folder, ItemType.Component, false))
            {
                var comp = (Component)MEngine.GetObject(item.Key.ToString());
                var configType = item.Value.ToLower();
                if (configType == BootstrapFilename)
                {
                    //There is a dummy _All component used to publish a bootstrap file containing a list of other files that have been generated
                    allComponent = comp;
                    continue;
                }
                var settings = new Dictionary<string, List<string>>();
                if (configType.Equals(TemplateConfigName) || configType.Equals(SchemasConfigName) || configType.Equals(TaxonomiesConfigName) || configType.Equals(MappingsConfigName))
                {
                    //template, schema and taxonomy config is only published from the master web publication/default localization
                    if (IsMasterWebPublication())
                    {
                        switch (configType)
                        {
                            case TemplateConfigName:
                                isArray = false;
                                settings = ReadTemplateData();
                                break;
                            case SchemasConfigName:
                                isArray = false;
                                settings = ReadSchemaData();
                                break;
                            case TaxonomiesConfigName:
                                isArray = false;
                                settings = ReadTaxonomiesData();
                                break;
                            case MappingsConfigName:
                                isArray = true;
                                settings = ReadMappingsData();
                                break;
                        }
                    }
                }
                else
                {
                    settings = new Dictionary<string, List<string>> { { configType, ReadComponentData(comp) } };
                }
                //Add all the JSON files to the package and publish them as binaries
                foreach (var key in settings.Keys)
                {
                    Item jsonItem;
                    if (isArray)
                    {
                        jsonItem = MPackage.CreateStringItem(ContentType.Text, String.Format("[{0}]", String.Join(",\n", settings[key])));
                    }
                    else
                    {
                        jsonItem = MPackage.CreateStringItem(ContentType.Text, String.Format("{{{0}}}", String.Join(",\n", settings[key])));
                    }
                    var filename = key + JsonExtension;
                    MPackage.PushItem(type + "/" + filename, jsonItem);
                    var binary = MEngine.PublishingContext.RenderedItem.AddBinary(jsonItem.GetAsStream(), filename, sg, key, comp, JsonMimetype);
                    files.Add(Json.Encode(binary.Url));
                }
            }
            if (allComponent == null)
            {
                throw new Exception(String.Format("Cannot find '_All' component. Please ensure that this component is in the {0} folder.", folder.WebDavUrl));
            }
            //We create a special asset file _all.js which contains the path of all other asset files of this type that have been published
            Item filesItem = MPackage.CreateStringItem(ContentType.Text, String.Format("{{\"defaultLocalization\":{0},\"files\":[{1}]}}", Json.Encode(IsMasterWebPublication()), String.Join(",", files)));
            MPackage.PushItem(type + "/" + BootstrapFilename + JsonExtension, filesItem);
            var bootstrapBinary = MEngine.PublishingContext.RenderedItem.AddBinary(filesItem.GetAsStream(), BootstrapFilename + JsonExtension, sg, type, allComponent, JsonMimetype);
            return bootstrapBinary.Url;
        }

        private static List<string> ReadComponentData(Component comp)
        {
            var fields = new ItemFields(comp.Content, comp.Schema);
            var settings = new List<string>();
            var configFields = fields.GetEmbeddedFields("settings");
            if (configFields.Any())
            {
                //either schema is a generic multival embedded name/value
                foreach (var setting in configFields)
                {
                    settings.Add(String.Format("{0}:{1}", Json.Encode(setting.GetTextValue("name")), Json.Encode(setting.GetTextValue("value"))));
                }
            }
            else
            {
                //... or its a custom schema with individual fields
                foreach (var field in fields)
                {
                    //TODO handle more types
                    if (field is TextField)
                    {
                        settings.Add(String.Format("{0}:{1}", Json.Encode(field.Name), Json.Encode(fields.GetTextValue(field.Name))));
                    }
                }
            }
            return settings;
        }

        private Dictionary<string, List<string>> ReadMappingsData()
        {
            bool containsTsiVocabulary = false;

            // generate a list of vocabulary prefix and name from appdata
            var res = new Dictionary<string, List<string>> { { VocabulariesConfigName, new List<string>() } };
            ApplicationData globalAppData = MEngine.GetSession().SystemManager.LoadGlobalApplicationData(VocabulariesAppDataId);
            if (globalAppData != null)
            {
                XElement vocabulariesXml = XElement.Parse(Encoding.Unicode.GetString(globalAppData.Data));
                foreach (var vocabulary in vocabulariesXml.Elements())
                {
                    string prefix = vocabulary.Attribute("prefix").Value;
                    res[VocabulariesConfigName].Add(String.Format("{{\"Prefix\":{0},\"Vocab\":{1}}}", Json.Encode(prefix), Json.Encode(vocabulary.Attribute("name").Value)));
                    if (prefix.Equals(TsiPrefix))
                    {
                        containsTsiVocabulary = true;
                    }
                }
            }
            // add tsi vocabulary if it is not there already
            if (!containsTsiVocabulary)
            {
                res[VocabulariesConfigName].Add(String.Format("{{\"Prefix\":{0},\"Vocab\":{1}}}", Json.Encode(TsiPrefix), Json.Encode(TsiVocabulary)));
            }

            // generate a list of schema + id, separated by module
            var schemaFilter = new RepositoryItemsFilter(MEngine.GetSession())
            {
                Recursive = true,
                ItemTypes = new List<ItemType> { ItemType.Schema },
                BaseColumns = ListBaseColumns.Extended
            };
            foreach (XmlElement item in GetPublication().GetListItems(schemaFilter).ChildNodes)
            {
                var type = item.GetAttribute("Type");
                var subType = item.GetAttribute("SubType");
                // we only consider normal schemas (type=8 subtype=0)
                if ((type == "8" && subType == "0"))
                {
                    var id = item.GetAttribute("ID");
                    var schema = (Schema)MEngine.GetObject(id);
                    var module = GetModuleName(schema.WebDavUrl);
                    if (module != null)
                    {
                        var key = module + "." + SchemasConfigName;
                        if (!res.ContainsKey(key))
                        {
                            res.Add(key, new List<string>());
                        }

                        // add schema typeof using tridion standard implementation vocabulary prefix
                        string typeOf = string.Format("{0}:{1}", TsiPrefix, schema.RootElementName);
                        StringBuilder schemaSemantics = new StringBuilder();
                        // append schema typeof from appdata 
                        ApplicationData appData = schema.LoadApplicationData(TypeOfAppDataId);
                        if (appData != null)
                        {
                            typeOf += "," + Encoding.Unicode.GetString(appData.Data);
                        }
                        schemaSemantics.Append(BuildSchemaSemanticsJson(typeOf));

                        // TODO: serialize schema fields xml to json in a smart way
                        // field: {"Name":"something","IsMultiValue":true,"Semantics":[],"Fields":[]}
                        // field semantics: {"Prefix":"s","Entity";"Article","Property":"headline"}
                        StringBuilder fields = new StringBuilder();

                        // load namespace manager with schema namespaces
                        XmlNamespaceManager nsmgr = SchemaNamespaceManager(schema.Xsd.OwnerDocument.NameTable);

                        // build field elements from schema
                        fields.Append(BuildSchemaFieldsJson(schema, typeOf, nsmgr));

                        res[key].Add(string.Format("{{\"Id\":{0},\"RootElement\":{1},\"Fields\":[{2}],\"Semantics\":[{3}]}}", Json.Encode(schema.Id.ItemId), Json.Encode(schema.RootElementName), fields, schemaSemantics));
                    }
                }
            }
            return res;
        }

        private XmlNamespaceManager SchemaNamespaceManager(XmlNameTable nameTable)
        {
            // load namespace manager with schema namespaces
            XmlNamespaceManager nsmgr = new XmlNamespaceManager(nameTable);
            foreach (var item in _namespaces)
            {
                nsmgr.AddNamespace(item.Key, item.Value);
            }
            return nsmgr;
        }

        // field: {"Name":"something","IsMultiValue":true,"Semantics":[],"Fields":[]}
        // field semantics: {"Prefix":"s","Entity":"Article","Property":"headline"}
        private string BuildSchemaFieldsJson(Schema schema, string typeOf, XmlNamespaceManager nsmgr, bool embedded = false)
        {
            StringBuilder fields = new StringBuilder();

            // loop over all field elements in schema
            bool first = true;
            string xpath = string.Format("/xsd:schema/xsd:element[@name='{0}']/xsd:complexType/xsd:sequence/xsd:element", schema.RootElementName);
            if (embedded)
            {
                xpath = string.Format("/xsd:schema/xsd:complexType[@name='{0}']/xsd:sequence/xsd:element", schema.RootElementName);
            }
            foreach (XmlNode fieldNode in schema.Xsd.SelectNodes(xpath, nsmgr))
            {
                string fieldTypeOf = typeOf;
                StringBuilder fieldSemantics = new StringBuilder();

                // if maxOccurs is anything else than 1, it is a multi value field
                bool isMultiValue = !fieldNode.Attributes["maxOccurs"].Value.Equals("1");

                // read semantic mapping from field so we can append it to the schema typeof
                XmlNode typeOfNode = fieldNode.SelectSingleNode("xsd:annotation/xsd:appinfo/tcm:ExtensionXml/mapping:typeof", nsmgr);
                if (typeOfNode != null)
                {
                    fieldTypeOf = typeOf + "," + typeOfNode.InnerText;
                }

                // use field xml name as initial semantic mapping for field
                string property = string.Format("{0}:{1}", TsiPrefix, fieldNode.Attributes["name"].Value);
                // read semantic mapping from field and append if available
                XmlNode propertyNode = fieldNode.SelectSingleNode("xsd:annotation/xsd:appinfo/tcm:ExtensionXml/mapping:property", nsmgr);
                if (propertyNode != null)
                {
                    property += "," + propertyNode.InnerText;
                }
                fieldSemantics.Append(BuildFieldSemanticsJson(property, fieldTypeOf));

                // handle embedded fields
                StringBuilder embeddedFields = new StringBuilder();
                XmlNode embeddedSchemaNode = fieldNode.SelectSingleNode("xsd:annotation/xsd:appinfo/tcm:EmbeddedSchema", nsmgr);
                if (embeddedSchemaNode != null)
                {
                    string uri = embeddedSchemaNode.Attributes["href", "http://www.w3.org/1999/xlink"].Value;
                    Schema embeddedSchema = (Schema)MEngine.GetObject(uri);
                    string embeddedTypeOf = string.Format("{0}:{1}", TsiPrefix, embeddedSchema.RootElementName);
                    // append schema typeof from appdata for embedded schemas
                    ApplicationData appData = embeddedSchema.LoadApplicationData(TypeOfAppDataId);
                    if (appData != null)
                    {
                        embeddedTypeOf += "," + Encoding.Unicode.GetString(appData.Data);
                    }
                    embeddedFields.Append(BuildSchemaFieldsJson(embeddedSchema, embeddedTypeOf, nsmgr, true));
                }

                // TODO: handle link fields
                //XmlAttribute typeAttribute = fieldNode.Attributes["type"];
                //if (typeAttribute != null)
                //{
                //    bool isSimpleLink = typeAttribute.Value.Equals("tcmi:SimpleLink");
                //    XmlNode allowedTargetSchemasNode = fieldNode.SelectSingleNode("xsd:annotation/xsd:appinfo/tcm:AllowedTargetSchemas", nsmgr);
                //    if (allowedTargetSchemasNode != null)
                //    {
                //        foreach (XmlNode allowedTargetSchemaNode in allowedTargetSchemasNode.SelectNodes("tcm:TargetSchema", nsmgr))
                //        {
                //            string uri = allowedTargetSchemaNode.Attributes["href", "http://www.w3.org/1999/xlink"].Value;
                //            Schema allowedTargetSchema = (Schema)MEngine.GetObject(uri);
                //
                //        }
                //    }
                //    else
                //    {
                //        // if there are no allowed target schemas, all schemas are allowed...
                //    }
                //}
                //else
                //{
                //    // if there is no type attribute, it is not a simple type so look for <xsd:complexType> inside the element
                //}

                if (first)
                {
                    first = false;
                }
                else
                {
                    fields.Append(",");
                }
                fields.AppendFormat("{{\"Name\":{0},\"IsMultiValue\":{1},\"Semantics\":[{2}],\"Fields\":[{3}]}}",
                    Json.Encode(fieldNode.Attributes["name"].Value),
                    Json.Encode(isMultiValue), fieldSemantics, embeddedFields);
            }

            return fields.ToString();
        }

        // schema semantics: {"Prefix":"s","Entity":"Article"}
        private static string BuildSchemaSemanticsJson(string input)
        {
            StringBuilder semantics = new StringBuilder();
            if (!string.IsNullOrEmpty(input))
            {
                // input = "s:Article" but can also be "s:Article,x:Something"
                string[] values = input.Split(',');
                bool first = true;
                foreach (var value in values)
                {
                    if (first)
                    {
                        first = false;
                    }
                    else
                    {
                        semantics.Append(",");
                    }
                    string[] parts = value.Split(':');
                    semantics.AppendFormat("{{\"Prefix\":{0},\"Entity\":{1}}}", Json.Encode(parts[0]), Json.Encode(parts[1]));
                }
            }

            return semantics.ToString();
        }

        // field semantics: {"Prefix":"s","Entity":"Article","Property":"headline"}
        private static string BuildFieldSemanticsJson(string input, string entity)
        {
            Dictionary<string, string> entities = new Dictionary<string, string>();
            StringBuilder semantics = new StringBuilder();
            if (!string.IsNullOrEmpty(input))
            {
                // entity = "s:Article" but can also be "s:Article,x:Something"
                string[] values = entity.Split(',');
                foreach (var value in values)
                {
                    string[] parts = value.Split(':');
                    entities.Add(parts[0], parts[1]);
                }

                // input = "s:headline" but can also be "s:headline,x:something"
                string[] properties = input.Split(',');
                for (int index = 0; index < properties.Length; index++)
                {
                    if (index > 0)
                    {
                        semantics.Append(",");
                    }
                    var value = properties[index];
                    string[] parts = value.Split(':');
                    semantics.AppendFormat("{{\"Prefix\":{0},\"Entity\":{1},\"Property\":{2}}}", Json.Encode(parts[0]), entities[parts[0]], Json.Encode(parts[1]));
                }
            }

            return semantics.ToString();
        }

        private Dictionary<string, List<string>> ReadTaxonomiesData()
        {
            //Generate a list of taxonomy + id
            var res = new Dictionary<string, List<string>>();
            var settings = new List<string>();
            var taxFilter = new TaxonomiesFilter(MEngine.GetSession()) { BaseColumns = ListBaseColumns.Extended };
            foreach (XmlElement item in GetPublication().GetListTaxonomies(taxFilter).ChildNodes)
            {
                var id = item.GetAttribute("ID");
                var taxonomy = (Category)MEngine.GetObject(id);
                settings.Add(String.Format("{0}:{1}", Json.Encode(GetKeyFromTaxonomy(taxonomy)), Json.Encode(taxonomy.Id.ItemId)));
            }
            res.Add("core." + TaxonomiesConfigName, settings);
            return res;
        }

        private Dictionary<string, List<string>> ReadSchemaData()
{'fraction_non_alphanumeric': 0.10701264797227349, 'fraction_numerical': 0.003465814466399604, 'mean_word_length': 1.667967577304113, 'pattern_counts': {'":': 71, '<': 64, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 19, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1195038', 'n_tokens_mistral': 11064, 'n_tokens_neox': 10295, 'n_words': 2342}