<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.SharePoint.Client;
using Microsoft.SharePoint.Client.Taxonomy;
using OfficeDevPnP.Core.Diagnostics;
using OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers.TokenDefinitions;

namespace OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers
{
    internal class ObjectTermGroups : ObjectHandlerBase
    {
        private List<ReusedTerm> reusedTerms;

        public override string Name => "Term Groups";

        public override TokenParser ProvisionObjects(Web web, Model.ProvisioningTemplate template, TokenParser parser,
            ProvisioningTemplateApplyingInformation applyingInformation)
        {
            using (var scope = new PnPMonitoredScope(this.Name))
            {
                this.reusedTerms = new List<ReusedTerm>();

                TaxonomySession taxSession = TaxonomySession.GetTaxonomySession(web.Context);
                TermStore termStore = null;
                TermGroup siteCollectionTermGroup = null;

                try
                {
                    termStore = taxSession.GetDefaultKeywordsTermStore();
                    web.Context.Load(termStore,
                        ts => ts.DefaultLanguage,
                        ts => ts.Groups.Include(
                            tg => tg.Name,
                            tg => tg.Id,
                            tg => tg.TermSets.Include(
                                tset => tset.Name,
                                tset => tset.Id)));
                    siteCollectionTermGroup = termStore.GetSiteCollectionGroup((web.Context as ClientContext).Site, false);
                    web.Context.Load(siteCollectionTermGroup);
                    web.Context.ExecuteQueryRetry();
                }
                catch (ServerException)
                {
                    // If the GetDefaultSiteCollectionTermStore method call fails ... raise a specific Warning
                    WriteMessage(CoreResources.Provisioning_ObjectHandlers_TermGroups_Wrong_Configuration, ProvisioningMessageType.Warning);

                    // and exit skipping the current handler
                    return parser;
                }

                SiteCollectionTermGroupNameToken siteCollectionTermGroupNameToken =
                    new SiteCollectionTermGroupNameToken(web);

                foreach (var modelTermGroup in template.TermGroups)
                {
                    #region Group

                    var newGroup = false;
                    var normalizedGroupName = TaxonomyItem.NormalizeName(web.Context, modelTermGroup.Name);
                    web.Context.ExecuteQueryRetry();

                    TermGroup group = termStore.Groups.FirstOrDefault(
                        g => g.Id == modelTermGroup.Id || g.Name == normalizedGroupName.Value);
                    if (group == null)
                    {
                        if (modelTermGroup.Name == "Site Collection" ||
                            parser.ParseString(modelTermGroup.Name) ==
                            siteCollectionTermGroupNameToken.GetReplaceValue() ||
                            modelTermGroup.SiteCollectionTermGroup)
                        {
                            var site = (web.Context as ClientContext).Site;
                            group = termStore.GetSiteCollectionGroup(site, true);
                            web.Context.Load(group, g => g.Name, g => g.Id, g => g.TermSets.Include(
                                tset => tset.Name,
                                tset => tset.Id));
                            web.Context.ExecuteQueryRetry();
                        }
                        else
                        {
                            var parsedGroupName = parser.ParseString(modelTermGroup.Name);
                            var parsedNormalizedGroupName = TaxonomyItem.NormalizeName(web.Context, parsedGroupName);
                            web.Context.ExecuteQueryRetry();

                            group = termStore.Groups.FirstOrDefault(g => g.Name == parsedNormalizedGroupName.Value);

                            if (group == null)
                            {
                                if (modelTermGroup.Id == Guid.Empty)
                                {
                                    modelTermGroup.Id = Guid.NewGuid();
                                }
                                group = termStore.CreateGroup(parsedGroupName, modelTermGroup.Id);

                                group.Description = modelTermGroup.Description;

#if !ONPREMISES

                                // Handle TermGroup Contributors, if any
                                if (modelTermGroup.Contributors != null && modelTermGroup.Contributors.Count > 0)
                                {
                                    foreach (var c in modelTermGroup.Contributors)
                                    {
                                        group.AddContributor(c.Name);
                                    }
                                }

                                // Handle TermGroup Managers, if any
                                if (modelTermGroup.Managers != null && modelTermGroup.Managers.Count > 0)
                                {
                                    foreach (var m in modelTermGroup.Managers)
                                    {
                                        group.AddGroupManager(m.Name);
                                    }
                                }

#endif

                                termStore.CommitAll();
                                web.Context.Load(group);
                                web.Context.ExecuteQueryRetry();

                                newGroup = true;

                            }
                        }
                    }

                    #endregion

                    #region TermSets

                    foreach (var modelTermSet in modelTermGroup.TermSets)
                    {
                        TermSet set = null;
                        var newTermSet = false;

                        var normalizedTermSetName = TaxonomyItem.NormalizeName(web.Context, modelTermSet.Name);
                        web.Context.ExecuteQueryRetry();

                        if (!newGroup)
                        {
                            set =
                                group.TermSets.FirstOrDefault(
                                    ts => ts.Id == modelTermSet.Id || ts.Name == normalizedTermSetName.Value);
                        }
                        if (set == null)
                        {
                            if (modelTermSet.Id == Guid.Empty)
                            {
                                modelTermSet.Id = Guid.NewGuid();
                            }
                            set = group.CreateTermSet(parser.ParseString(modelTermSet.Name), modelTermSet.Id,
                                modelTermSet.Language ?? termStore.DefaultLanguage);
                            parser.AddToken(new TermSetIdToken(web, group.Name, modelTermSet.Name, modelTermSet.Id));
                            if (!siteCollectionTermGroup.ServerObjectIsNull.Value)
                            {
                                if (group.Name == siteCollectionTermGroup.Name)
                                {
                                    parser.AddToken((new SiteCollectionTermSetIdToken(web, modelTermSet.Name, modelTermSet.Id)));
                                }
                            }
                            newTermSet = true;
                            set.Description = modelTermSet.Description;
                            set.IsOpenForTermCreation = modelTermSet.IsOpenForTermCreation;
                            set.IsAvailableForTagging = modelTermSet.IsAvailableForTagging;
                            foreach (var property in modelTermSet.Properties)
                            {
                                set.SetCustomProperty(property.Key, parser.ParseString(property.Value));
                            }
                            if (modelTermSet.Owner != null)
                            {
                                set.Owner = modelTermSet.Owner;
                            }
                            termStore.CommitAll();
                            web.Context.Load(set);
                            web.Context.ExecuteQueryRetry();
                        }

                        web.Context.Load(set, s => s.Terms.Include(t => t.Id, t => t.Name));
                        web.Context.ExecuteQueryRetry();
                        var terms = set.Terms;

                        foreach (var modelTerm in modelTermSet.Terms)
                        {
                            if (!newTermSet)
                            {
                                if (terms.Any())
                                {
                                    var term = terms.FirstOrDefault(t => t.Id == modelTerm.Id);
                                    if (term == null)
                                    {
                                        var normalizedTermName = TaxonomyItem.NormalizeName(web.Context, modelTerm.Name);
                                        web.Context.ExecuteQueryRetry();

                                        term = terms.FirstOrDefault(t => t.Name == normalizedTermName.Value);
                                        if (term == null)
                                        {
                                            var returnTuple = CreateTerm<TermSet>(web, modelTerm, set, termStore, parser,
                                                scope);
                                            if (returnTuple != null)
                                            {
                                                modelTerm.Id = returnTuple.Item1;
                                                parser = returnTuple.Item2;
                                            }
                                        }
                                        else
                                        {
                                            modelTerm.Id = term.Id;
                                        }
                                    }
                                    else
                                    {
                                        modelTerm.Id = term.Id;
                                    }
                                }
                                else
                                {
                                    var returnTuple = CreateTerm<TermSet>(web, modelTerm, set, termStore, parser, scope);
                                    if (returnTuple != null)
                                    {
                                        modelTerm.Id = returnTuple.Item1;
                                        parser = returnTuple.Item2;
                                    }
                                }
                            }
                            else
                            {
                                var returnTuple = CreateTerm<TermSet>(web, modelTerm, set, termStore, parser, scope);
                                if (returnTuple != null)
                                {
                                    modelTerm.Id = returnTuple.Item1;
                                    parser = returnTuple.Item2;
                                }
                            }
                        }

                        // do we need custom sorting?
                        if (modelTermSet.Terms.Any(t => t.CustomSortOrder > -1))
                        {
                            var sortedTerms = modelTermSet.Terms.OrderBy(t => t.CustomSortOrder);

                            var customSortString = sortedTerms.Aggregate(string.Empty,
                                (a, i) => a + i.Id.ToString() + ":");
                            customSortString = customSortString.TrimEnd(new[] { ':' });

                            set.CustomSortOrder = customSortString;
                            termStore.CommitAll();
                            web.Context.ExecuteQueryRetry();
                        }
                    }

                    #endregion

                }

                foreach (var reusedTerm in this.reusedTerms)
                {
                    TryReuseTerm(web, reusedTerm.ModelTerm, reusedTerm.Parent, reusedTerm.TermStore, parser, scope);
                }
            }
            return parser;
        }

        public class ReusedTerm
        {
            public Model.Term ModelTerm { get; set; }
            public TaxonomyItem Parent { get; set; }
            public TermStore TermStore { get; set; }
        }

        private Tuple<Guid, TokenParser> CreateTerm<T>(Web web, Model.Term modelTerm, TaxonomyItem parent,
            TermStore termStore, TokenParser parser, PnPMonitoredScope scope) where T : TaxonomyItem
        {
            // If the term is a re-used term and the term is not a source term, skip for now and create later
            if (modelTerm.IsReused && !modelTerm.IsSourceTerm)
            {
                this.reusedTerms.Add(new ReusedTerm()
                {
                    ModelTerm = modelTerm,
                    Parent = parent,
                    TermStore = termStore
                });
                return null;
            }

            // Create new term
            Term term;
            if (modelTerm.Id == Guid.Empty)
            {
                modelTerm.Id = Guid.NewGuid();
            }

            if (parent is Term)
            {
                term = ((Term)parent).CreateTerm(parser.ParseString(modelTerm.Name), modelTerm.Language ?? termStore.DefaultLanguage, modelTerm.Id);
            }
            else
            {
                term = ((TermSet)parent).CreateTerm(parser.ParseString(modelTerm.Name), modelTerm.Language ?? termStore.DefaultLanguage, modelTerm.Id);
            }
            if (!string.IsNullOrEmpty(modelTerm.Description))
            {
                term.SetDescription(modelTerm.Description, modelTerm.Language ?? termStore.DefaultLanguage);
            }
            if (!string.IsNullOrEmpty(modelTerm.Owner))
            {
                term.Owner = modelTerm.Owner;
            }

            term.IsAvailableForTagging = modelTerm.IsAvailableForTagging;

            if (modelTerm.Properties.Any() || modelTerm.Labels.Any() || modelTerm.LocalProperties.Any())
            {<commit_msg>Added parsing of description resource to TermGroup, TermSet and Term.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.SharePoint.Client;
using Microsoft.SharePoint.Client.Taxonomy;
using OfficeDevPnP.Core.Diagnostics;
using OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers.TokenDefinitions;

namespace OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers
{
    internal class ObjectTermGroups : ObjectHandlerBase
    {
        private List<ReusedTerm> reusedTerms;

        public override string Name => "Term Groups";

        public override TokenParser ProvisionObjects(Web web, Model.ProvisioningTemplate template, TokenParser parser,
            ProvisioningTemplateApplyingInformation applyingInformation)
        {
            using (var scope = new PnPMonitoredScope(this.Name))
            {
                this.reusedTerms = new List<ReusedTerm>();

                TaxonomySession taxSession = TaxonomySession.GetTaxonomySession(web.Context);
                TermStore termStore = null;
                TermGroup siteCollectionTermGroup = null;

                try
                {
                    termStore = taxSession.GetDefaultKeywordsTermStore();
                    web.Context.Load(termStore,
                        ts => ts.DefaultLanguage,
                        ts => ts.Groups.Include(
                            tg => tg.Name,
                            tg => tg.Id,
                            tg => tg.TermSets.Include(
                                tset => tset.Name,
                                tset => tset.Id)));
                    siteCollectionTermGroup = termStore.GetSiteCollectionGroup((web.Context as ClientContext).Site, false);
                    web.Context.Load(siteCollectionTermGroup);
                    web.Context.ExecuteQueryRetry();
                }
                catch (ServerException)
                {
                    // If the GetDefaultSiteCollectionTermStore method call fails ... raise a specific Warning
                    WriteMessage(CoreResources.Provisioning_ObjectHandlers_TermGroups_Wrong_Configuration, ProvisioningMessageType.Warning);

                    // and exit skipping the current handler
                    return parser;
                }

                SiteCollectionTermGroupNameToken siteCollectionTermGroupNameToken =
                    new SiteCollectionTermGroupNameToken(web);

                foreach (var modelTermGroup in template.TermGroups)
                {
                    #region Group

                    var newGroup = false;
                    var normalizedGroupName = TaxonomyItem.NormalizeName(web.Context, modelTermGroup.Name);
                    web.Context.ExecuteQueryRetry();

                    TermGroup group = termStore.Groups.FirstOrDefault(
                        g => g.Id == modelTermGroup.Id || g.Name == normalizedGroupName.Value);
                    if (group == null)
                    {
                        var parsedGroupName = parser.ParseString(modelTermGroup.Name);
                        var parsedDescription = parser.ParseString(modelTermGroup.Description);

                        if (modelTermGroup.Name == "Site Collection" ||
                            parsedGroupName == siteCollectionTermGroupNameToken.GetReplaceValue() ||
                            modelTermGroup.SiteCollectionTermGroup)
                        {
                            var site = (web.Context as ClientContext).Site;
                            group = termStore.GetSiteCollectionGroup(site, true);
                            web.Context.Load(group, g => g.Name, g => g.Id, g => g.TermSets.Include(
                                tset => tset.Name,
                                tset => tset.Id));
                            web.Context.ExecuteQueryRetry();
                        }
                        else
                        {
                            var parsedNormalizedGroupName = TaxonomyItem.NormalizeName(web.Context, parsedGroupName);
                            web.Context.ExecuteQueryRetry();

                            group = termStore.Groups.FirstOrDefault(g => g.Name == parsedNormalizedGroupName.Value);

                            if (group == null)
                            {
                                if (modelTermGroup.Id == Guid.Empty)
                                {
                                    modelTermGroup.Id = Guid.NewGuid();
                                }
                                group = termStore.CreateGroup(parsedGroupName, modelTermGroup.Id);

                                group.Description = parsedDescription;

#if !ONPREMISES

                                // Handle TermGroup Contributors, if any
                                if (modelTermGroup.Contributors != null && modelTermGroup.Contributors.Count > 0)
                                {
                                    foreach (var c in modelTermGroup.Contributors)
                                    {
                                        group.AddContributor(c.Name);
                                    }
                                }

                                // Handle TermGroup Managers, if any
                                if (modelTermGroup.Managers != null && modelTermGroup.Managers.Count > 0)
                                {
                                    foreach (var m in modelTermGroup.Managers)
                                    {
                                        group.AddGroupManager(m.Name);
                                    }
                                }

#endif

                                termStore.CommitAll();
                                web.Context.Load(group);
                                web.Context.ExecuteQueryRetry();

                                newGroup = true;

                            }
                        }
                    }

                    #endregion

                    #region TermSets

                    foreach (var modelTermSet in modelTermGroup.TermSets)
                    {
                        TermSet set = null;
                        var newTermSet = false;

                        var normalizedTermSetName = TaxonomyItem.NormalizeName(web.Context, modelTermSet.Name);
                        web.Context.ExecuteQueryRetry();

                        if (!newGroup)
                        {
                            set =
                                group.TermSets.FirstOrDefault(
                                    ts => ts.Id == modelTermSet.Id || ts.Name == normalizedTermSetName.Value);
                        }
                        if (set == null)
                        {
                            if (modelTermSet.Id == Guid.Empty)
                            {
                                modelTermSet.Id = Guid.NewGuid();
                            }
                            set = group.CreateTermSet(parser.ParseString(modelTermSet.Name), modelTermSet.Id,
                                modelTermSet.Language ?? termStore.DefaultLanguage);
                            parser.AddToken(new TermSetIdToken(web, group.Name, modelTermSet.Name, modelTermSet.Id));
                            if (!siteCollectionTermGroup.ServerObjectIsNull.Value)
                            {
                                if (group.Name == siteCollectionTermGroup.Name)
                                {
                                    parser.AddToken((new SiteCollectionTermSetIdToken(web, modelTermSet.Name, modelTermSet.Id)));
                                }
                            }
                            newTermSet = true;
                            set.Description = parser.ParseString(modelTermSet.Description);
                            set.IsOpenForTermCreation = modelTermSet.IsOpenForTermCreation;
                            set.IsAvailableForTagging = modelTermSet.IsAvailableForTagging;
                            foreach (var property in modelTermSet.Properties)
                            {
                                set.SetCustomProperty(property.Key, parser.ParseString(property.Value));
                            }
                            if (modelTermSet.Owner != null)
                            {
                                set.Owner = modelTermSet.Owner;
                            }
                            termStore.CommitAll();
                            web.Context.Load(set);
                            web.Context.ExecuteQueryRetry();
                        }

                        web.Context.Load(set, s => s.Terms.Include(t => t.Id, t => t.Name));
                        web.Context.ExecuteQueryRetry();
                        var terms = set.Terms;

                        foreach (var modelTerm in modelTermSet.Terms)
                        {
                            if (!newTermSet)
                            {
                                if (terms.Any())
                                {
                                    var term = terms.FirstOrDefault(t => t.Id == modelTerm.Id);
                                    if (term == null)
                                    {
                                        var normalizedTermName = TaxonomyItem.NormalizeName(web.Context, modelTerm.Name);
                                        web.Context.ExecuteQueryRetry();

                                        term = terms.FirstOrDefault(t => t.Name == normalizedTermName.Value);
                                        if (term == null)
                                        {
                                            var returnTuple = CreateTerm<TermSet>(web, modelTerm, set, termStore, parser,
                                                scope);
                                            if (returnTuple != null)
                                            {
                                                modelTerm.Id = returnTuple.Item1;
                                                parser = returnTuple.Item2;
                                            }
                                        }
                                        else
                                        {
                                            modelTerm.Id = term.Id;
                                        }
                                    }
                                    else
                                    {
                                        modelTerm.Id = term.Id;
                                    }
                                }
                                else
                                {
                                    var returnTuple = CreateTerm<TermSet>(web, modelTerm, set, termStore, parser, scope);
                                    if (returnTuple != null)
                                    {
                                        modelTerm.Id = returnTuple.Item1;
                                        parser = returnTuple.Item2;
                                    }
                                }
                            }
                            else
                            {
                                var returnTuple = CreateTerm<TermSet>(web, modelTerm, set, termStore, parser, scope);
                                if (returnTuple != null)
                                {
                                    modelTerm.Id = returnTuple.Item1;
                                    parser = returnTuple.Item2;
                                }
                            }
                        }

                        // do we need custom sorting?
                        if (modelTermSet.Terms.Any(t => t.CustomSortOrder > -1))
                        {
                            var sortedTerms = modelTermSet.Terms.OrderBy(t => t.CustomSortOrder);

                            var customSortString = sortedTerms.Aggregate(string.Empty,
                                (a, i) => a + i.Id.ToString() + ":");
                            customSortString = customSortString.TrimEnd(new[] { ':' });

                            set.CustomSortOrder = customSortString;
                            termStore.CommitAll();
                            web.Context.ExecuteQueryRetry();
                        }
                    }

                    #endregion

                }

                foreach (var reusedTerm in this.reusedTerms)
                {
                    TryReuseTerm(web, reusedTerm.ModelTerm, reusedTerm.Parent, reusedTerm.TermStore, parser, scope);
                }
            }
            return parser;
        }

        public class ReusedTerm
        {
            public Model.Term ModelTerm { get; set; }
            public TaxonomyItem Parent { get; set; }
            public TermStore TermStore { get; set; }
        }

        private Tuple<Guid, TokenParser> CreateTerm<T>(Web web, Model.Term modelTerm, TaxonomyItem parent,
            TermStore termStore, TokenParser parser, PnPMonitoredScope scope) where T : TaxonomyItem
        {
            // If the term is a re-used term and the term is not a source term, skip for now and create later
            if (modelTerm.IsReused && !modelTerm.IsSourceTerm)
            {
                this.reusedTerms.Add(new ReusedTerm()
                {
                    ModelTerm = modelTerm,
                    Parent = parent,
                    TermStore = termStore
                });
                return null;
            }

            // Create new term
            Term term;
            if (modelTerm.Id == Guid.Empty)
            {
                modelTerm.Id = Guid.NewGuid();
            }

            if (parent is Term)
            {
                term = ((Term)parent).CreateTerm(parser.ParseString(modelTerm.Name), modelTerm.Language ?? termStore.DefaultLanguage, modelTerm.Id);
            }
            else
            {
                term = ((TermSet)parent).CreateTerm(parser.ParseString(modelTerm.Name), modelTerm.Language ?? termStore.DefaultLanguage, modelTerm.Id);
            }
            if (!string.IsNullOrEmpty(modelTerm.Description))
            {
                term.SetDescription(parser.ParseString(modelTerm.Description), modelTerm.Language ?? termStore.DefaultLanguage);
            }
            if (!string.IsNullOrEmpty(modelTerm.Owner))
            {
                term.Owner = modelTerm.Owner;
            }

            term.IsAvailableForTagging = modelTerm.IsAvailableForTagging;

            if (modelTerm.Properties.Any() || modelTerm.Labels.Any() || modelTerm.LocalProperties.Any())
            {
{'fraction_non_alphanumeric': 0.06983145303792307, 'fraction_numerical': 0.0006116623623759684, 'mean_word_length': 0.915576384820673, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 132, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5631824', 'n_tokens_mistral': 6198, 'n_tokens_neox': 5826, 'n_words': 1106}