<commit_before>﻿using System;
using System.Collections.Generic;
using System.Data.SqlTypes;
using System.Linq;
using System.Text;
using System.Web.Configuration;
using System.Web.Instrumentation;
using System.Xml.Linq;
using Microsoft.SharePoint.Client;
using OfficeDevPnP.Core.Framework.ObjectHandlers;
using OfficeDevPnP.Core.Framework.ObjectHandlers.TokenDefinitions;
using OfficeDevPnP.Core.Framework.Provisioning.Model;
using Field = Microsoft.SharePoint.Client.Field;
using View = OfficeDevPnP.Core.Framework.Provisioning.Model.View;

namespace OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers
{
    public class ObjectListInstance : ObjectHandlerBase
    {
        public override void ProvisionObjects(Web web, ProvisioningTemplate template)
        {
            //var parser = new TokenParser(web);

            if (!web.IsPropertyAvailable("ServerRelativeUrl"))
            {
                web.Context.Load(web, w => w.ServerRelativeUrl);
                web.Context.ExecuteQueryRetry();
            }

            web.Context.Load(web.Lists, lc => lc.IncludeWithDefaultProperties(l => l.RootFolder.ServerRelativeUrl));
            web.Context.ExecuteQueryRetry();
            var existingLists = web.Lists.Select(existingList => existingList.RootFolder.ServerRelativeUrl).ToList();
            var serverRelativeUrl = web.ServerRelativeUrl;

            var createdLists = new List<ListInfo>();

            #region Lists

            foreach (var list in template.Lists)
            {
                if (!existingLists.Contains(UrlUtility.Combine(serverRelativeUrl, list.Url)))
                {
                    var listCreate = new ListCreationInformation();
                    listCreate.Description = list.Description;
                    listCreate.TemplateType = list.TemplateType;
                    listCreate.Title = list.Title;
                    listCreate.QuickLaunchOption = list.OnQuickLaunch ? QuickLaunchOptions.On : QuickLaunchOptions.Off;
                    listCreate.Url = list.Url.ToParsedString();
                    listCreate.TemplateFeatureId = list.TemplateFeatureID;
                    var createdList = web.Lists.Add(listCreate);

                    createdList.EnableVersioning = list.EnableVersioning;
                    if (!String.IsNullOrEmpty(list.DocumentTemplate))
                    {
                        createdList.DocumentTemplateUrl = list.DocumentTemplate.ToParsedString();
                    }
                    createdList.Hidden = list.Hidden;
                    createdList.ContentTypesEnabled = list.ContentTypesEnabled;

                    createdList.Update();

                    web.Context.Load(createdList.Views);
                    web.Context.Load(createdList, l => l.Id);
                    web.Context.Load(createdList, l => l.RootFolder.ServerRelativeUrl);
                    web.Context.Load(createdList.ContentTypes);
                    web.Context.ExecuteQueryRetry();

                    if (list.RemoveExistingContentTypes)
                    {
                        while (createdList.ContentTypes.Any())
                        {
                            createdList.ContentTypes[0].DeleteObject();
                        }
                        web.Context.ExecuteQueryRetry();
                    }

                    foreach (var ctBinding in list.ContentTypeBindings)
                    {
                        createdList.AddContentTypeToListById(ctBinding.ContentTypeID);
                        if (ctBinding.Default)
                        {
                            createdList.SetDefaultContentTypeToList(ctBinding.ContentTypeID);
                        }
                    }
                    createdLists.Add(new ListInfo { CreatedList = createdList, ListInstance = list });

                    TokenParser.AddToken(new ListIdToken(web, list.Title, createdList.Id));

                    TokenParser.AddToken(new ListUrlToken(web, list.Title, createdList.RootFolder.ServerRelativeUrl.Substring(web.ServerRelativeUrl.Length + 1)));


                }

            }

            #endregion

            #region Fields
            foreach (var listInfo in createdLists)
            {
                if (listInfo.ListInstance.Fields.Any())
                {
                    foreach (var field in listInfo.ListInstance.Fields)
                    {
                        XElement fieldElement = XElement.Parse(field.SchemaXml.ToParsedString());
                        var id = fieldElement.Attribute("ID").Value;

                        Guid fieldGuid = Guid.Empty;
                        if (Guid.TryParse(id, out fieldGuid))
                        {
                            if (!listInfo.CreatedList.FieldExistsById(fieldGuid))
                            {
                                var listIdentifier = fieldElement.Attribute("List") != null ? fieldElement.Attribute("List").Value : null;

                                if (listIdentifier != null)
                                {
                                    // Temporary remove list attribute from fieldElement
                                    fieldElement.Attribute("List").Remove();
                                }

                                var fieldXml = fieldElement.ToString();
                                listInfo.CreatedList.Fields.AddFieldAsXml(fieldXml, false, AddFieldOptions.DefaultValue);
                            }
                        }
                    }
                }
                listInfo.CreatedList.Update();
                web.Context.ExecuteQueryRetry();
            }

            #endregion


            #region FieldRefs

            foreach (var listInfo in createdLists)
            {

                if (listInfo.ListInstance.FieldRefs.Any())
                {
                    foreach (var fieldRef in listInfo.ListInstance.FieldRefs)
                    {
                        var field = web.GetFieldById<Field>(fieldRef.ID);
                        if (!listInfo.CreatedList.FieldExistsById(fieldRef.ID))
                        {
                            var createdField = listInfo.CreatedList.Fields.Add(field);
                            if (!string.IsNullOrEmpty(fieldRef.DisplayName))
                            {
                                createdField.Title = fieldRef.DisplayName;
                                createdField.Update();
                            }
                        }

                    }<commit_msg>Added handling for hidden and required fieldrefs<commit_after>﻿using System;
using System.Collections.Generic;
using System.Data.SqlTypes;
using System.Linq;
using System.Text;
using System.Web.Configuration;
using System.Web.Instrumentation;
using System.Xml.Linq;
using Microsoft.SharePoint.Client;
using OfficeDevPnP.Core.Framework.ObjectHandlers;
using OfficeDevPnP.Core.Framework.ObjectHandlers.TokenDefinitions;
using OfficeDevPnP.Core.Framework.Provisioning.Model;
using Field = Microsoft.SharePoint.Client.Field;
using View = OfficeDevPnP.Core.Framework.Provisioning.Model.View;

namespace OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers
{
    public class ObjectListInstance : ObjectHandlerBase
    {
        public override void ProvisionObjects(Web web, ProvisioningTemplate template)
        {
            //var parser = new TokenParser(web);

            if (!web.IsPropertyAvailable("ServerRelativeUrl"))
            {
                web.Context.Load(web, w => w.ServerRelativeUrl);
                web.Context.ExecuteQueryRetry();
            }

            web.Context.Load(web.Lists, lc => lc.IncludeWithDefaultProperties(l => l.RootFolder.ServerRelativeUrl));
            web.Context.ExecuteQueryRetry();
            var existingLists = web.Lists.Select(existingList => existingList.RootFolder.ServerRelativeUrl).ToList();
            var serverRelativeUrl = web.ServerRelativeUrl;

            var createdLists = new List<ListInfo>();

            #region Lists

            foreach (var list in template.Lists)
            {
                if (!existingLists.Contains(UrlUtility.Combine(serverRelativeUrl, list.Url)))
                {
                    var listCreate = new ListCreationInformation();
                    listCreate.Description = list.Description;
                    listCreate.TemplateType = list.TemplateType;
                    listCreate.Title = list.Title;
                    listCreate.QuickLaunchOption = list.OnQuickLaunch ? QuickLaunchOptions.On : QuickLaunchOptions.Off;
                    listCreate.Url = list.Url.ToParsedString();
                    listCreate.TemplateFeatureId = list.TemplateFeatureID;
                    var createdList = web.Lists.Add(listCreate);

                    createdList.EnableVersioning = list.EnableVersioning;
                    if (!String.IsNullOrEmpty(list.DocumentTemplate))
                    {
                        createdList.DocumentTemplateUrl = list.DocumentTemplate.ToParsedString();
                    }
                    createdList.Hidden = list.Hidden;
                    createdList.ContentTypesEnabled = list.ContentTypesEnabled;

                    createdList.Update();

                    web.Context.Load(createdList.Views);
                    web.Context.Load(createdList, l => l.Id);
                    web.Context.Load(createdList, l => l.RootFolder.ServerRelativeUrl);
                    web.Context.Load(createdList.ContentTypes);
                    web.Context.ExecuteQueryRetry();

                    if (list.RemoveExistingContentTypes)
                    {
                        while (createdList.ContentTypes.Any())
                        {
                            createdList.ContentTypes[0].DeleteObject();
                        }
                        web.Context.ExecuteQueryRetry();
                    }

                    foreach (var ctBinding in list.ContentTypeBindings)
                    {
                        createdList.AddContentTypeToListById(ctBinding.ContentTypeID);
                        if (ctBinding.Default)
                        {
                            createdList.SetDefaultContentTypeToList(ctBinding.ContentTypeID);
                        }
                    }
                    createdLists.Add(new ListInfo { CreatedList = createdList, ListInstance = list });

                    TokenParser.AddToken(new ListIdToken(web, list.Title, createdList.Id));

                    TokenParser.AddToken(new ListUrlToken(web, list.Title, createdList.RootFolder.ServerRelativeUrl.Substring(web.ServerRelativeUrl.Length + 1)));


                }

            }

            #endregion

            #region Fields
            foreach (var listInfo in createdLists)
            {
                if (listInfo.ListInstance.Fields.Any())
                {
                    foreach (var field in listInfo.ListInstance.Fields)
                    {
                        XElement fieldElement = XElement.Parse(field.SchemaXml.ToParsedString());
                        var id = fieldElement.Attribute("ID").Value;

                        Guid fieldGuid = Guid.Empty;
                        if (Guid.TryParse(id, out fieldGuid))
                        {
                            if (!listInfo.CreatedList.FieldExistsById(fieldGuid))
                            {
                                var listIdentifier = fieldElement.Attribute("List") != null ? fieldElement.Attribute("List").Value : null;

                                if (listIdentifier != null)
                                {
                                    // Temporary remove list attribute from fieldElement
                                    fieldElement.Attribute("List").Remove();
                                }

                                var fieldXml = fieldElement.ToString();
                                listInfo.CreatedList.Fields.AddFieldAsXml(fieldXml, false, AddFieldOptions.DefaultValue);
                            }
                        }
                    }
                }
                listInfo.CreatedList.Update();
                web.Context.ExecuteQueryRetry();
            }

            #endregion


            #region FieldRefs

            foreach (var listInfo in createdLists)
            {

                if (listInfo.ListInstance.FieldRefs.Any())
                {
                    foreach (var fieldRef in listInfo.ListInstance.FieldRefs)
                    {
                        var field = web.GetFieldById<Field>(fieldRef.ID);
                        if (!listInfo.CreatedList.FieldExistsById(fieldRef.ID))
                        {
                            var createdField = listInfo.CreatedList.Fields.Add(field);
                            if (!string.IsNullOrEmpty(fieldRef.DisplayName))
                            {
                                createdField.Title = fieldRef.DisplayName;
                            }
                            createdField.Hidden = fieldRef.Hidden;
                            createdField.Required = fieldRef.Required;

                            createdField.Update();
                        }

                    }
{'fraction_non_alphanumeric': 0.07940334488473708, 'fraction_numerical': 0.00030134096730450503, 'mean_word_length': 1.3944805194805194, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1378794', 'n_tokens_mistral': 3077, 'n_tokens_neox': 2862, 'n_words': 474}