<commit_before>﻿using Microsoft.Online.SharePoint.TenantAdministration;
using Microsoft.SharePoint.Client;
using Microsoft.SharePoint.Client.Publishing;
using Microsoft.SharePoint.Client.Search.Query;
using OfficeDevPnP.Core.Entities;
using OfficeDevPnP.Core.Utilities;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace Microsoft.SharePoint.Client
{
    /// <summary>
    /// Class that deals with site (both site collection and web site) creation, status, retrieval and settings
    /// </summary>
    public static class SiteExtensions
    {
        const string MSG_CONTEXT_CLOSED = "ClientContext gets closed after action is completed. Calling ExecuteQuery again returns an error. Verify that you have an open ClientContext object.";
        const string SITE_STATUS_ACTIVE = "Active";
        const string SITE_STATUS_CREATING = "Creating";
        const string SITE_STATUS_RECYCLED = "Recycled";
        const string INDEXED_PROPERTY_KEY = "vti_indexedpropertykeys";

        #region Site (collection) query, creation and deletion

        /// <summary>
        /// Adds a SiteEntity by launching site collection creation and waits for the creation to finish
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="properties">Describes the site collection to be created</param>
        /// <param name="removeSiteFromRecycleBin">It true and site is present in recycle bin, it will be removed first from the recycle bin</param>
        /// <param name="wait">If true, processing will halt until the site collection has been created</param>
        /// <returns>Guid of the created site collection</returns>
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2200:RethrowToPreserveStackDetails")]
        public static Guid AddSiteCollectionTenant(this Web web, SiteEntity properties, bool removeFromRecycleBin = false, bool wait = true)
        {
            if (removeFromRecycleBin)
            {
                if (CheckIfSiteExistsInTenant(web, properties.Url, SITE_STATUS_RECYCLED))
                {
                    web.DeleteSiteCollectionFromRecycleBinTenant(properties.Url);
                }
            }

            Tenant tenant = new Tenant(web.Context);
            SiteCreationProperties newsite = new SiteCreationProperties();
            newsite.Url = properties.Url;
            newsite.Owner = properties.SiteOwnerLogin;
            newsite.Template = properties.Template;
            newsite.Title = properties.Title;
            newsite.StorageMaximumLevel = properties.StorageMaximumLevel;
            newsite.StorageWarningLevel = properties.StorageWarningLevel;
            newsite.TimeZoneId = properties.TimeZoneId;
            newsite.UserCodeMaximumLevel = properties.UserCodeMaximumLevel;
            newsite.UserCodeWarningLevel = properties.UserCodeWarningLevel;
            newsite.Lcid = properties.Lcid;

            try
            {
                SpoOperation op = tenant.CreateSite(newsite);
                web.Context.Load(tenant);
                web.Context.Load(op, i => i.IsComplete, i => i.PollingInterval);
                web.Context.ExecuteQuery();

                if (wait)
                {
                    //check if site creation operation is complete
                    while (!op.IsComplete)
                    {
                        System.Threading.Thread.Sleep(op.PollingInterval);
                        op.RefreshLoad();
                        if (!op.IsComplete)
                        {
                            try
                            {
                                web.Context.ExecuteQuery();
                            }
                            catch (WebException webEx)
                            {
                                // Context connection gets closed after action completed.
                                // Calling ExecuteQuery again returns an error which can be ignored
                                LoggingUtility.LogWarning(MSG_CONTEXT_CLOSED, webEx, EventCategory.Site);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                // Eat the siteSubscription exception to make the same code work for MT as on-prem April 2014 CU+
                if (ex.Message.IndexOf("Parameter name: siteSubscription") == -1)
                {
                    throw ex;
                }
            }

            // Get site guid and return
            var siteGuid = web.GetSiteGuidByUrlTenant(new Uri(properties.Url));

            return siteGuid;
        }

        /// <summary>
        /// Returns if a site collection is in a particular status. If the url contains a sub site then returns true is the sub site exists, false if not. 
        /// Status is irrelevant for sub sites
        /// </summary>
        /// <param name="web">Tenant admin web</param>
        /// <param name="siteUrl">Url to the site collection</param>
        /// <param name="status">Status to check (Active, Creating, Recycled)</param>
        /// <returns>True if in status, false if not in status</returns>
        public static bool CheckIfSiteExistsInTenant(this Web web, string siteUrl, string status)
        {
            bool ret = false;
            //Get the site name
            var url = new Uri(siteUrl);
            var UrlDomain = string.Format("{0}://{1}", url.Scheme, url.Host);
            int idx = url.PathAndQuery.Substring(1).IndexOf("/") + 2;
            var UrlPath = url.PathAndQuery.Substring(0, idx);
            var Name = url.PathAndQuery.Substring(idx);
            var index = Name.IndexOf('/');
            Tenant tenant = new Tenant(web.Context);

            //Judge whether this site collection is existing or not
            if (index == -1)
            {
                var properties = tenant.GetSitePropertiesByUrl(siteUrl, false);
                web.Context.Load(properties);
                web.Context.ExecuteQuery();
                ret = properties.Status.Equals(status, StringComparison.OrdinalIgnoreCase);
            }
            //Judge whether this sub web site is existing or not
            else
            {
                var site = tenant.GetSiteByUrl(string.Format(System.Globalization.CultureInfo.CurrentCulture, "{0}{1}{2}", UrlDomain, UrlPath, Name.Split("/".ToCharArray())[0]));
                var subweb = site.OpenWeb(Name.Substring(index + 1));
                web.Context.Load(subweb, w => w.Title);
                web.Context.ExecuteQuery();
                ret = true;
            }
            return ret;
        }

        /// <summary>
        /// Launches a site collection creation and waits for the creation to finish 
        /// </summary>
        /// <param name="web">Context to admin site</param>
        /// <param name="url">The SPO url</param>
        /// <param name="title">The site title</param>
        /// <param name="siteOwnerLogin">Owner account</param>
        /// <param name="template">Site template being used</param>
        /// <param name="storageMaximumLevel">Site quota in MB</param>
        /// <param name="storageWarningLevel">Site quota warning level in MB</param>
        /// <param name="timeZoneId">TimeZoneID for the site. "(UTC+01:00) Brussels, Copenhagen, Madrid, Paris" = 3 </param>
        /// <param name="userCodeMaximumLevel">The user code quota in points</param>
        /// <param name="userCodeWarningLevel">The user code quota warning level in points</param>
        /// <param name="lcid">The site locale. See http://technet.microsoft.com/en-us/library/ff463597.aspx for a complete list of Lcid's</param>
        /// <returns></returns>
        public static Guid CreateSiteCollectionTenant(this Web web, string url, string title, string siteOwnerLogin,
                                                        string template, int storageMaximumLevel, int storageWarningLevel,
                                                        int timeZoneId, int userCodeMaximumLevel, int userCodeWarningLevel,
                                                        uint lcid, bool removeFromRecycleBin = false, bool wait = true)
        {
            SiteEntity siteCol = new SiteEntity()
            {
                Url = url,
                Title = title,
                SiteOwnerLogin = siteOwnerLogin,
                Template = template,
                StorageMaximumLevel = storageMaximumLevel,
                StorageWarningLevel = storageWarningLevel,
                TimeZoneId = timeZoneId,
                UserCodeMaximumLevel = userCodeMaximumLevel,
                UserCodeWarningLevel = userCodeWarningLevel,
                Lcid = lcid
            };

            return AddSiteCollectionTenant(web, siteCol, removeFromRecycleBin, wait);
        }

        /// <summary>
        /// Deletes a site collection
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">Url of the site collection to delete</param>
        /// <param name="useRecycleBin">Leave the deleted site collection in the site collection recycle bin</param>
        /// <returns>True if deleted</returns>
        public static bool DeleteSiteCollectionTenant(this Web web, string siteUrl, bool useRecycleBin)
        {
            bool ret = false;
            Tenant tenant = new Tenant(web.Context);
            SpoOperation op = tenant.RemoveSite(siteUrl);
            web.Context.Load(tenant);
            web.Context.Load(op, i => i.IsComplete, i => i.PollingInterval);
            web.Context.ExecuteQuery();

            //check if site creation operation is complete
            while (!op.IsComplete)
            {
                System.Threading.Thread.Sleep(op.PollingInterval);
                op.RefreshLoad();
                if (!op.IsComplete)
                {
                    try
                    {
                        web.Context.ExecuteQuery();
                    }
                    catch (WebException webEx)
                    {
                        // Context connection gets closed after action completed.
                        // Calling ExecuteQuery again returns an error which can be ignored
                        LoggingUtility.LogWarning(MSG_CONTEXT_CLOSED, webEx, EventCategory.Site);
                    }
                }
            }

            if (useRecycleBin)
            {
                return true;
            }

            // To delete Site collection completely, (may take a longer time)
            op = tenant.RemoveDeletedSite(siteUrl);
            web.Context.Load(op, i => i.IsComplete, i => i.PollingInterval);
            web.Context.ExecuteQuery();

            while (!op.IsComplete)
            {
                System.Threading.Thread.Sleep(op.PollingInterval);
                op.RefreshLoad();
                if (!op.IsComplete)
                {
                    try
                    {
                        web.Context.ExecuteQuery();
                    }
                    catch (WebException webEx)
                    {
                        // Context connection gets closed after action completed.
                        // Calling ExecuteQuery again returns an error which can be ignored
                        LoggingUtility.LogWarning(MSG_CONTEXT_CLOSED, webEx, EventCategory.Site);
                    }
                }
            }

            ret = true;
            return ret;
        }

        /// <summary>
        /// Deletes a site collection from the site collection recycle bin
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">URL of the site collection to delete</param>
        /// <returns>True if deleted</returns>
        public static bool DeleteSiteCollectionFromRecycleBinTenant(this Web web, string siteUrl)
        {
            bool ret = false;
            Tenant tenant = new Tenant(web.Context);
            SpoOperation op = tenant.RemoveDeletedSite(siteUrl);
            web.Context.Load(op, i => i.IsComplete, i => i.PollingInterval);
            web.Context.ExecuteQuery();

            while (!op.IsComplete)
            {
                System.Threading.Thread.Sleep(op.PollingInterval);
                op.RefreshLoad();
                if (!op.IsComplete)
                {
                    try
                    {
                        web.Context.ExecuteQuery();
                    }
                    catch (WebException webEx)
                    {
                        // Context connection gets closed after action completed.
                        // Calling ExecuteQuery again returns an error which can be ignored
                        LoggingUtility.LogWarning(MSG_CONTEXT_CLOSED, webEx, EventCategory.Site);
                    }
                }
            }

            ret = true;
            return ret;
        }

        /// <summary>
        /// Checks if a site collection exists
        /// </summary>
        /// <param name="web">Tenant admin web</param>
        /// <param name="siteUrl">URL to the site collection</param>
        /// <returns>True if existing, false if not</returns>
        public static bool DoesSiteExistInTenant(this Web web, string siteUrl)
        {
            try
            {
                return web.CheckIfSiteExistsInTenant(siteUrl, SITE_STATUS_ACTIVE) ||
                       web.CheckIfSiteExistsInTenant(siteUrl, SITE_STATUS_CREATING) ||
                       web.CheckIfSiteExistsInTenant(siteUrl, SITE_STATUS_RECYCLED);
            }
            catch (Exception ex)
            {
                if (ex is Microsoft.SharePoint.Client.ServerException && (ex.Message.IndexOf("Unable to access site") != -1 || ex.Message.IndexOf("Cannot get site") != -1))
                {
                    return true;
                }
                else
                    LoggingUtility.LogError("Could not determine if site exists in tenant.", ex, EventCategory.Site);

                return false;
            }
        }

        /// <summary>
        /// Gets the ID of site collection with specified URL
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">A URL that specifies a site collection to get ID.</param>
        /// <returns>The Guid of a site collection</returns>
        public static Guid GetSiteGuidByUrlTenant(this Web web, string siteUrl)
        {
            if (!string.IsNullOrEmpty(siteUrl))
                throw new ArgumentNullException("siteUrl");

            return web.GetSiteGuidByUrlTenant(new Uri(siteUrl));
        }

        /// <summary>
        /// Gets the ID of site collection with specified URL
        /// </summary>
        /// <param name="web">Tenant admin web</param>
        /// <param name="siteUrl">A URL that specifies a site collection to get ID.</param>
        /// <returns>The Guid of a site collection</returns>
        public static Guid GetSiteGuidByUrlTenant(this Web web, Uri siteUrl)
        {
            Guid siteGuid = Guid.Empty;

            Site site = null;
            Tenant tenant = new Tenant(web.Context);
            site = tenant.GetSiteByUrl(siteUrl.OriginalString);
            web.Context.Load(site);
            web.Context.ExecuteQuery();
            siteGuid = site.Id;

            return siteGuid;
        }

        /// <summary>
        /// Returns available webtemplates/site definitions
        /// </summary>
        /// <param name="web">Site to be processed - needs to be tenant site admin site</param>
        /// <param name="lcid"></param>
        /// <param name="compatibilityLevel">14 for SharePoint 2010, 15 for SharePoint 2013/SharePoint Online</param>
        /// <returns></returns>
        public static SPOTenantWebTemplateCollection GetWebTemplatesTenant(this Web web, uint lcid, int compatibilityLevel)
        {
            Tenant tenant = new Tenant(web.Context);

            var templates = tenant.GetSPOTenantWebTemplates(lcid, compatibilityLevel);

            web.Context.Load(templates);

            web.Context.ExecuteQuery();

            return templates;
        }

        /// <summary>
        /// Checks if a site collection is Active
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">URL to the site collection</param>
        /// <returns>True if active, false if not</returns>
        public static bool IsSiteActiveTenant(this Web web, string siteUrl)
        {
            try
            {
                return web.CheckIfSiteExistsInTenant(siteUrl, "Active");
            }
            catch (Exception ex)
            {
                if (ex.Message.StartsWith("Cannot get site"))
                {
                    return false;
                }
                LoggingUtility.LogError("Error finding if site is active tenant.", ex, EventCategory.Site);
                throw;
            }
        }

        /// <summary>
        /// Checks if a site collection exists, relies on tenant admin API
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">URL to the site collection</param>
        /// <returns>True if existing, false if not</returns>
        public static bool SiteExistsInTenant(this Web web, string siteUrl)
        {
            try
            {
                //Get the site name
                Tenant tenant = new Tenant(web.Context);
                var properties = tenant.GetSitePropertiesByUrl(siteUrl, false);
                web.Context.Load(properties);
                web.Context.ExecuteQuery();

                // Will cause an exception if site URL is not there. Not optimal, but the way it works.
                return true;
            }
            catch (Exception ex)
            {
                if (ex is Microsoft.SharePoint.Client.ServerException && (ex.Message.IndexOf("Unable to access site") != -1 || ex.Message.IndexOf("Cannot get site") != -1))
                {
                    return false;
                }
                else
                {
                    return true;
                }
            }
        }

        /// <summary>
        /// Checks if a sub site exists
        /// </summary>
        /// <param name="web">Tenant admin web</param>
        /// <param name="siteUrl">URL to the sub site</param>
        /// <returns>True if existing, false if not</returns>
        public static bool SubSiteExistsInTenant(this Web web, string siteUrl)
        {
            try
            {
                return web.CheckIfSiteExistsInTenant(siteUrl, "");
            }
            catch (Exception ex)
            {
                if (ex is Microsoft.SharePoint.Client.ServerException && (ex.Message.IndexOf("Unable to access site") != -1 || ex.Message.IndexOf("Cannot get site") != -1))
                {
                    return true;
                }
                else
                {
                    return false;
                }
            }
        }

        #endregion

        #region Web (site) query, creation and deletion

        /// <summary>
        /// Adds a sub site to an existing site
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="parent">Information about the parent site</param>
        /// <param name="subsite">Information describing the sub site to be added</param>
        /// <param name="inheritPermissions">Does the sub site inherit the permissions of the parent site</param>
        /// <param name="inheritNavigation">Does the sub site inherit the navigation of the parent site</param>
        [Obsolete("Should use CreateWeb(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static void AddSite(this Web web, SiteEntity parent, SiteEntity subsite, bool inheritPermissions, bool inheritNavigation)
        {
            // Call actual implementation
            CreateWeb(web, subsite.Title, subsite.Url, subsite.Description, subsite.Template, (int)subsite.Lcid, inheritPermissions, inheritNavigation);
        }

        /// <summary>
        /// Adds a sub site to an existing site
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="title">Title for the site</param>
        /// <param name="description">Description for the new site</param>
        /// <param name="template">Template for the site, like STS#0</param>
        /// <param name="language">Language code for the site, like 1033</param>
        /// <param name="inheritPermissions">Should the new site inherit permissions</param>
        /// <param name="inheritNavigation">Should the new site inherent navigation</param>
        [Obsolete("Should use CreateWeb(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static void AddSite(this Web web, string title, string url, string description, string template, uint language, bool inheritPermissions, bool inheritNavigation)
        {
            // Call centralized route to call internal creation logic
            CreateWeb(web, title, url, description, template, (int)language, inheritPermissions, inheritNavigation);
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="web"></param>
        /// <param name="subsite"></param>
        /// <param name="inheritPermissions"></param>
        /// <param name="inheritNavigation"></param>
        [Obsolete("Should use CreateWeb(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static Web CreateSite(this Web web, SiteEntity subsite, bool inheritPermissions = true, bool inheritNavigation = true)
        {
            // Call actual implementation
            return CreateWeb(web, subsite.Title, subsite.Url, subsite.Description, subsite.Template, (int)subsite.Lcid, inheritPermissions, inheritNavigation);
        }

        [Obsolete("Should use CreateWeb(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static Web CreateSite(this Web web, string title, string url, string description, string template, int language, bool inheritPermissions = true, bool inheritNavigation = true)
        {
            return CreateWeb(web, title, url, description, template, language, inheritPermissions, inheritNavigation);
        }

        /// <summary>
        /// Adds a new child Web (site) to a parent Web.
        /// </summary>
        /// <param name="parentWeb">The parent Web (site) to create under</param>
        /// <param name="subsite">Details of the Web (site) to add. Only Title, Url (as the leaf URL), Description, Template and Language are used.</param>
        /// <param name="inheritPermissions">Specifies whether the new site will inherit permissions from its parent site.</param>
        /// <param name="inheritNavigation">Specifies whether the site inherits navigation.</param>
        /// <returns></returns>
        public static Web CreateWeb(this Web parentWeb, SiteEntity subsite, bool inheritPermissions = true, bool inheritNavigation = true)
        {
            return CreateWeb(parentWeb, subsite.Title, subsite.Url, subsite.Description, subsite.Template, (int)subsite.Lcid, inheritPermissions, inheritNavigation);
        }

        /// <summary>
        /// Adds a new child Web (site) to a parent Web.
        /// </summary>
        /// <param name="parentWeb">The parent Web (site) to create under</param>
        /// <param name="title">The title of the new site. </param>
        /// <param name="leafUrl">A string that represents the URL leaf name.</param>
        /// <param name="description">The description of the new site. </param>
        /// <param name="template">The name of the site template to be used for creating the new site. </param>
        /// <param name="language">The locale ID that specifies the language of the new site. </param>
        /// <param name="inheritPermissions">Specifies whether the new site will inherit permissions from its parent site.</param>
        /// <param name="inheritNavigation">Specifies whether the site inherits navigation.</param>
        public static Web CreateWeb(this Web parentWeb, string title, string leafUrl, string description, string template, int language, bool inheritPermissions = true, bool inheritNavigation = true)
        {
            // TODO: Check for any other illegal characters in SharePoint
            if (leafUrl.Contains('/') || leafUrl.Contains('\\'))
            {
                throw new ArgumentException("The argument must be a single web URL and cannot contain path characters.", "leafUrl");
            }
            LoggingUtility.Internal.TraceInformation((int)EventId.CreateWeb, "Creating web '{0}' with template '{1}'.", leafUrl, template);
            WebCreationInformation creationInfo = new WebCreationInformation()
            {
                Url = leafUrl,
                Title = title,
                Description = description,
                UseSamePermissionsAsParentSite = inheritPermissions,
                WebTemplate = template,
                Language = language
            };

            Web newWeb = parentWeb.Webs.Add(creationInfo);
            newWeb.Navigation.UseShared = inheritNavigation;
            newWeb.Update();

            parentWeb.Context.ExecuteQuery();

            return newWeb;
        }

        /// <summary>
        /// Deletes the child website with the specified leaf URL, from a parent Web, if it exists. 
        /// </summary>
        /// <param name="parentWeb">The parent Web (site) to delete from</param>
        /// <param name="leafUrl">A string that represents the URL leaf name.</param>
        /// <returns>true if the web was deleted; otherwise false if nothing was done</returns>
        public static bool DeleteWeb(this Web parentWeb, string leafUrl)
        {
            // TODO: Check for any other illegal characters in SharePoint
            if (leafUrl.Contains('/') || leafUrl.Contains('\\'))
            {
                throw new ArgumentException("The argument must be a single web URL and cannot contain path characters.", "leafUrl");
            }
            var deleted = false;
            Utility.EnsureWeb(parentWeb.Context, parentWeb, "ServerRelativeUrl");
            var serverRelativeUrl = UrlUtility.Combine(parentWeb.ServerRelativeUrl, leafUrl);
            var webs = parentWeb.Webs;
            // NOTE: Predicate does not take into account a required case-insensitive comparison
            //var results = parentWeb.Context.LoadQuery<Web>(webs.Where(item => item.ServerRelativeUrl == serverRelativeUrl));
            parentWeb.Context.Load(webs);
            parentWeb.Context.ExecuteQuery();
            var existingWeb = webs.FirstOrDefault(item => string.Equals(item.ServerRelativeUrl, serverRelativeUrl, StringComparison.OrdinalIgnoreCase));
            if (existingWeb != null)
            {
                LoggingUtility.Internal.TraceInformation((int)EventId.DeleteWeb, "Deleting web '{0}'.", serverRelativeUrl);
                existingWeb.DeleteObject();
                parentWeb.Context.ExecuteQuery();
                deleted = true;
            }
            else
            {
                LoggingUtility.Internal.TraceVerbose("Delete requested but web '{0}' not found, nothing to do.", serverRelativeUrl);
            }
            return deleted;
        }

        /// <summary>
        /// Gets the collection of the URLs of all Web sites that are contained within the site collection, 
        /// including the top-level site and its subsites.
        /// </summary>
        /// <param name="site">Site collection to retrieve the URLs for.</param>
        /// <returns>An enumeration containing the full URLs as strings.</returns>
        /// <remarks>
        /// <para>
        /// This is analagous to the <code>SPSite.AllWebs</code> property and can be used to get a collection
        /// of all web site URLs to loop through, e.g. for branding.
        /// </para>
        /// </remarks>
        public static IEnumerable<string> GetAllWebUrls(this Site site)
        {
            var siteContext = site.Context;
            siteContext.Load(site, s => s.Url);
            siteContext.ExecuteQuery();
            var queue = new Queue<string>();
            queue.Enqueue(site.Url);
            while (queue.Count > 0)
            {
                var currentUrl = queue.Dequeue();
                using (var webContext = new ClientContext(currentUrl))
                {
                    webContext.Credentials = siteContext.Credentials;
                    webContext.Load(webContext.Web, web => web.Webs);
                    webContext.ExecuteQuery();
                    foreach (var subWeb in webContext.Web.Webs)
                    {
                        queue.Enqueue(subWeb.Url);
                    }
                }
                yield return currentUrl;
            }
        }

        /// <summary>
        /// Checks if a site collection exists
        /// </summary>
        /// <param name="web">Site object opened with credentials that are reused for the site collection check</param>
        /// <param name="siteUrl">Fully qualified URL to the sub site</param>
        /// <returns>true if exists, false otherwise</returns>
        [Obsolete("Should use Context.WebExistsFullUrl(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static bool SiteExists(this Web web, string siteUrl)
        {
            return WebExistsFullUrl(web.Context, siteUrl);
        }

        /// <summary>
        /// Checks if a subsite exists
        /// </summary>
        /// <param name="web">Site object opened with credentials that are reused for the sub site check</param>
        /// <param name="siteUrl">Fully qualified URL to the sub site</param>
        /// <returns>true if exists, false otherwise</returns>
        [Obsolete("Should use Context.WebExists(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static bool SubSiteExists(this Web web, string siteUrl)
        {
            return WebExistsFullUrl(web.Context, siteUrl);
        }

        [Obsolete("Should use WebExists(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static bool SubSiteExistsWithUrl(this Web web, string url)
        {
            return WebExists(web, url);
        }

        /// <summary>
        /// Determines if a child Web site with the specified leaf URL exists. 
        /// </summary>
        /// <param name="parentWeb">The Web site to check under</param>
        /// <param name="leafUrl">A string that represents the URL leaf name.</param>
        /// <returns>true if the Web (site) exists; otherwise false</returns>
        public static bool WebExists(this Web parentWeb, string leafUrl)
        {
            // TODO: Check for any other illegal characters in SharePoint
            if (leafUrl.Contains('/') || leafUrl.Contains('\\'))
            {
                throw new ArgumentException("The argument must be a single web URL and cannot contain path characters.", "leafUrl");
            }

            Utility.EnsureWeb(parentWeb.Context, parentWeb, "ServerRelativeUrl");
            var serverRelativeUrl = UrlUtility.Combine(parentWeb.ServerRelativeUrl, leafUrl);
            var webs = parentWeb.Webs;
            // NOTE: Predicate does not take into account a required case-insensitive comparison
            //var results = parentWeb.Context.LoadQuery<Web>(webs.Where(item => item.ServerRelativeUrl == serverRelativeUrl));
            parentWeb.Context.Load(webs);
            parentWeb.Context.ExecuteQuery();
            var exists = webs.Any(item => string.Equals(item.ServerRelativeUrl, serverRelativeUrl, StringComparison.OrdinalIgnoreCase));
            return exists;
        }

        /// <summary>
        /// Determines if a Web (site) exists at the specified full URL, either accessible or that returns an access error.
        /// </summary>
        /// <param name="context">Existing context, used to provide credentials.</param>
        /// <param name="webFullUrl">Full URL of the site to check.</param>
        /// <returns>true if the Web (site) exists; otherwise false</returns>
        public static bool WebExistsFullUrl(this ClientRuntimeContext context, string webFullUrl)
        {
            bool exists = false;
            try
            {
                using(ClientContext testContext = new ClientContext(webFullUrl))
                {
                    testContext.Credentials = context.Credentials;
                    testContext.Load(testContext.Web, w => w.Title);
                    testContext.ExecuteQuery();
                    exists = true;
                }
            }
            catch (Exception ex)
            {
                if (ex is Microsoft.SharePoint.Client.ServerException &&
                    (ex.Message.IndexOf("Unable to access site") != -1 ||
                     ex.Message.IndexOf("Cannot get site") != -1))
                {
                    // Site exists, but you don't have access .. not sure if this is really valid<commit_msg>Specify only the properties needed to be retrieved<commit_after>﻿using Microsoft.Online.SharePoint.TenantAdministration;
using Microsoft.SharePoint.Client;
using Microsoft.SharePoint.Client.Publishing;
using Microsoft.SharePoint.Client.Search.Query;
using OfficeDevPnP.Core.Entities;
using OfficeDevPnP.Core.Utilities;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace Microsoft.SharePoint.Client
{
    /// <summary>
    /// Class that deals with site (both site collection and web site) creation, status, retrieval and settings
    /// </summary>
    public static class SiteExtensions
    {
        const string MSG_CONTEXT_CLOSED = "ClientContext gets closed after action is completed. Calling ExecuteQuery again returns an error. Verify that you have an open ClientContext object.";
        const string SITE_STATUS_ACTIVE = "Active";
        const string SITE_STATUS_CREATING = "Creating";
        const string SITE_STATUS_RECYCLED = "Recycled";
        const string INDEXED_PROPERTY_KEY = "vti_indexedpropertykeys";

        #region Site (collection) query, creation and deletion

        /// <summary>
        /// Adds a SiteEntity by launching site collection creation and waits for the creation to finish
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="properties">Describes the site collection to be created</param>
        /// <param name="removeSiteFromRecycleBin">It true and site is present in recycle bin, it will be removed first from the recycle bin</param>
        /// <param name="wait">If true, processing will halt until the site collection has been created</param>
        /// <returns>Guid of the created site collection</returns>
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2200:RethrowToPreserveStackDetails")]
        public static Guid AddSiteCollectionTenant(this Web web, SiteEntity properties, bool removeFromRecycleBin = false, bool wait = true)
        {
            if (removeFromRecycleBin)
            {
                if (CheckIfSiteExistsInTenant(web, properties.Url, SITE_STATUS_RECYCLED))
                {
                    web.DeleteSiteCollectionFromRecycleBinTenant(properties.Url);
                }
            }

            Tenant tenant = new Tenant(web.Context);
            SiteCreationProperties newsite = new SiteCreationProperties();
            newsite.Url = properties.Url;
            newsite.Owner = properties.SiteOwnerLogin;
            newsite.Template = properties.Template;
            newsite.Title = properties.Title;
            newsite.StorageMaximumLevel = properties.StorageMaximumLevel;
            newsite.StorageWarningLevel = properties.StorageWarningLevel;
            newsite.TimeZoneId = properties.TimeZoneId;
            newsite.UserCodeMaximumLevel = properties.UserCodeMaximumLevel;
            newsite.UserCodeWarningLevel = properties.UserCodeWarningLevel;
            newsite.Lcid = properties.Lcid;

            try
            {
                SpoOperation op = tenant.CreateSite(newsite);
                web.Context.Load(tenant);
                web.Context.Load(op, i => i.IsComplete, i => i.PollingInterval);
                web.Context.ExecuteQuery();

                if (wait)
                {
                    //check if site creation operation is complete
                    while (!op.IsComplete)
                    {
                        System.Threading.Thread.Sleep(op.PollingInterval);
                        op.RefreshLoad();
                        if (!op.IsComplete)
                        {
                            try
                            {
                                web.Context.ExecuteQuery();
                            }
                            catch (WebException webEx)
                            {
                                // Context connection gets closed after action completed.
                                // Calling ExecuteQuery again returns an error which can be ignored
                                LoggingUtility.LogWarning(MSG_CONTEXT_CLOSED, webEx, EventCategory.Site);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                // Eat the siteSubscription exception to make the same code work for MT as on-prem April 2014 CU+
                if (ex.Message.IndexOf("Parameter name: siteSubscription") == -1)
                {
                    throw ex;
                }
            }

            // Get site guid and return
            var siteGuid = web.GetSiteGuidByUrlTenant(new Uri(properties.Url));

            return siteGuid;
        }

        /// <summary>
        /// Returns if a site collection is in a particular status. If the url contains a sub site then returns true is the sub site exists, false if not. 
        /// Status is irrelevant for sub sites
        /// </summary>
        /// <param name="web">Tenant admin web</param>
        /// <param name="siteUrl">Url to the site collection</param>
        /// <param name="status">Status to check (Active, Creating, Recycled)</param>
        /// <returns>True if in status, false if not in status</returns>
        public static bool CheckIfSiteExistsInTenant(this Web web, string siteUrl, string status)
        {
            bool ret = false;
            //Get the site name
            var url = new Uri(siteUrl);
            var UrlDomain = string.Format("{0}://{1}", url.Scheme, url.Host);
            int idx = url.PathAndQuery.Substring(1).IndexOf("/") + 2;
            var UrlPath = url.PathAndQuery.Substring(0, idx);
            var Name = url.PathAndQuery.Substring(idx);
            var index = Name.IndexOf('/');
            Tenant tenant = new Tenant(web.Context);

            //Judge whether this site collection is existing or not
            if (index == -1)
            {
                var properties = tenant.GetSitePropertiesByUrl(siteUrl, false);
                web.Context.Load(properties);
                web.Context.ExecuteQuery();
                ret = properties.Status.Equals(status, StringComparison.OrdinalIgnoreCase);
            }
            //Judge whether this sub web site is existing or not
            else
            {
                var site = tenant.GetSiteByUrl(string.Format(System.Globalization.CultureInfo.CurrentCulture, "{0}{1}{2}", UrlDomain, UrlPath, Name.Split("/".ToCharArray())[0]));
                var subweb = site.OpenWeb(Name.Substring(index + 1));
                web.Context.Load(subweb, w => w.Title);
                web.Context.ExecuteQuery();
                ret = true;
            }
            return ret;
        }

        /// <summary>
        /// Launches a site collection creation and waits for the creation to finish 
        /// </summary>
        /// <param name="web">Context to admin site</param>
        /// <param name="url">The SPO url</param>
        /// <param name="title">The site title</param>
        /// <param name="siteOwnerLogin">Owner account</param>
        /// <param name="template">Site template being used</param>
        /// <param name="storageMaximumLevel">Site quota in MB</param>
        /// <param name="storageWarningLevel">Site quota warning level in MB</param>
        /// <param name="timeZoneId">TimeZoneID for the site. "(UTC+01:00) Brussels, Copenhagen, Madrid, Paris" = 3 </param>
        /// <param name="userCodeMaximumLevel">The user code quota in points</param>
        /// <param name="userCodeWarningLevel">The user code quota warning level in points</param>
        /// <param name="lcid">The site locale. See http://technet.microsoft.com/en-us/library/ff463597.aspx for a complete list of Lcid's</param>
        /// <returns></returns>
        public static Guid CreateSiteCollectionTenant(this Web web, string url, string title, string siteOwnerLogin,
                                                        string template, int storageMaximumLevel, int storageWarningLevel,
                                                        int timeZoneId, int userCodeMaximumLevel, int userCodeWarningLevel,
                                                        uint lcid, bool removeFromRecycleBin = false, bool wait = true)
        {
            SiteEntity siteCol = new SiteEntity()
            {
                Url = url,
                Title = title,
                SiteOwnerLogin = siteOwnerLogin,
                Template = template,
                StorageMaximumLevel = storageMaximumLevel,
                StorageWarningLevel = storageWarningLevel,
                TimeZoneId = timeZoneId,
                UserCodeMaximumLevel = userCodeMaximumLevel,
                UserCodeWarningLevel = userCodeWarningLevel,
                Lcid = lcid
            };

            return AddSiteCollectionTenant(web, siteCol, removeFromRecycleBin, wait);
        }

        /// <summary>
        /// Deletes a site collection
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">Url of the site collection to delete</param>
        /// <param name="useRecycleBin">Leave the deleted site collection in the site collection recycle bin</param>
        /// <returns>True if deleted</returns>
        public static bool DeleteSiteCollectionTenant(this Web web, string siteUrl, bool useRecycleBin)
        {
            bool ret = false;
            Tenant tenant = new Tenant(web.Context);
            SpoOperation op = tenant.RemoveSite(siteUrl);
            web.Context.Load(tenant);
            web.Context.Load(op, i => i.IsComplete, i => i.PollingInterval);
            web.Context.ExecuteQuery();

            //check if site creation operation is complete
            while (!op.IsComplete)
            {
                System.Threading.Thread.Sleep(op.PollingInterval);
                op.RefreshLoad();
                if (!op.IsComplete)
                {
                    try
                    {
                        web.Context.ExecuteQuery();
                    }
                    catch (WebException webEx)
                    {
                        // Context connection gets closed after action completed.
                        // Calling ExecuteQuery again returns an error which can be ignored
                        LoggingUtility.LogWarning(MSG_CONTEXT_CLOSED, webEx, EventCategory.Site);
                    }
                }
            }

            if (useRecycleBin)
            {
                return true;
            }

            // To delete Site collection completely, (may take a longer time)
            op = tenant.RemoveDeletedSite(siteUrl);
            web.Context.Load(op, i => i.IsComplete, i => i.PollingInterval);
            web.Context.ExecuteQuery();

            while (!op.IsComplete)
            {
                System.Threading.Thread.Sleep(op.PollingInterval);
                op.RefreshLoad();
                if (!op.IsComplete)
                {
                    try
                    {
                        web.Context.ExecuteQuery();
                    }
                    catch (WebException webEx)
                    {
                        // Context connection gets closed after action completed.
                        // Calling ExecuteQuery again returns an error which can be ignored
                        LoggingUtility.LogWarning(MSG_CONTEXT_CLOSED, webEx, EventCategory.Site);
                    }
                }
            }

            ret = true;
            return ret;
        }

        /// <summary>
        /// Deletes a site collection from the site collection recycle bin
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">URL of the site collection to delete</param>
        /// <returns>True if deleted</returns>
        public static bool DeleteSiteCollectionFromRecycleBinTenant(this Web web, string siteUrl)
        {
            bool ret = false;
            Tenant tenant = new Tenant(web.Context);
            SpoOperation op = tenant.RemoveDeletedSite(siteUrl);
            web.Context.Load(op, i => i.IsComplete, i => i.PollingInterval);
            web.Context.ExecuteQuery();

            while (!op.IsComplete)
            {
                System.Threading.Thread.Sleep(op.PollingInterval);
                op.RefreshLoad();
                if (!op.IsComplete)
                {
                    try
                    {
                        web.Context.ExecuteQuery();
                    }
                    catch (WebException webEx)
                    {
                        // Context connection gets closed after action completed.
                        // Calling ExecuteQuery again returns an error which can be ignored
                        LoggingUtility.LogWarning(MSG_CONTEXT_CLOSED, webEx, EventCategory.Site);
                    }
                }
            }

            ret = true;
            return ret;
        }

        /// <summary>
        /// Checks if a site collection exists
        /// </summary>
        /// <param name="web">Tenant admin web</param>
        /// <param name="siteUrl">URL to the site collection</param>
        /// <returns>True if existing, false if not</returns>
        public static bool DoesSiteExistInTenant(this Web web, string siteUrl)
        {
            try
            {
                return web.CheckIfSiteExistsInTenant(siteUrl, SITE_STATUS_ACTIVE) ||
                       web.CheckIfSiteExistsInTenant(siteUrl, SITE_STATUS_CREATING) ||
                       web.CheckIfSiteExistsInTenant(siteUrl, SITE_STATUS_RECYCLED);
            }
            catch (Exception ex)
            {
                if (ex is Microsoft.SharePoint.Client.ServerException && (ex.Message.IndexOf("Unable to access site") != -1 || ex.Message.IndexOf("Cannot get site") != -1))
                {
                    return true;
                }
                else
                    LoggingUtility.LogError("Could not determine if site exists in tenant.", ex, EventCategory.Site);

                return false;
            }
        }

        /// <summary>
        /// Gets the ID of site collection with specified URL
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">A URL that specifies a site collection to get ID.</param>
        /// <returns>The Guid of a site collection</returns>
        public static Guid GetSiteGuidByUrlTenant(this Web web, string siteUrl)
        {
            if (!string.IsNullOrEmpty(siteUrl))
                throw new ArgumentNullException("siteUrl");

            return web.GetSiteGuidByUrlTenant(new Uri(siteUrl));
        }

        /// <summary>
        /// Gets the ID of site collection with specified URL
        /// </summary>
        /// <param name="web">Tenant admin web</param>
        /// <param name="siteUrl">A URL that specifies a site collection to get ID.</param>
        /// <returns>The Guid of a site collection</returns>
        public static Guid GetSiteGuidByUrlTenant(this Web web, Uri siteUrl)
        {
            Guid siteGuid = Guid.Empty;

            Site site = null;
            Tenant tenant = new Tenant(web.Context);
            site = tenant.GetSiteByUrl(siteUrl.OriginalString);
            web.Context.Load(site);
            web.Context.ExecuteQuery();
            siteGuid = site.Id;

            return siteGuid;
        }

        /// <summary>
        /// Returns available webtemplates/site definitions
        /// </summary>
        /// <param name="web">Site to be processed - needs to be tenant site admin site</param>
        /// <param name="lcid"></param>
        /// <param name="compatibilityLevel">14 for SharePoint 2010, 15 for SharePoint 2013/SharePoint Online</param>
        /// <returns></returns>
        public static SPOTenantWebTemplateCollection GetWebTemplatesTenant(this Web web, uint lcid, int compatibilityLevel)
        {
            Tenant tenant = new Tenant(web.Context);

            var templates = tenant.GetSPOTenantWebTemplates(lcid, compatibilityLevel);

            web.Context.Load(templates);

            web.Context.ExecuteQuery();

            return templates;
        }

        /// <summary>
        /// Checks if a site collection is Active
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">URL to the site collection</param>
        /// <returns>True if active, false if not</returns>
        public static bool IsSiteActiveTenant(this Web web, string siteUrl)
        {
            try
            {
                return web.CheckIfSiteExistsInTenant(siteUrl, "Active");
            }
            catch (Exception ex)
            {
                if (ex.Message.StartsWith("Cannot get site"))
                {
                    return false;
                }
                LoggingUtility.LogError("Error finding if site is active tenant.", ex, EventCategory.Site);
                throw;
            }
        }

        /// <summary>
        /// Checks if a site collection exists, relies on tenant admin API
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="siteUrl">URL to the site collection</param>
        /// <returns>True if existing, false if not</returns>
        public static bool SiteExistsInTenant(this Web web, string siteUrl)
        {
            try
            {
                //Get the site name
                Tenant tenant = new Tenant(web.Context);
                var properties = tenant.GetSitePropertiesByUrl(siteUrl, false);
                web.Context.Load(properties);
                web.Context.ExecuteQuery();

                // Will cause an exception if site URL is not there. Not optimal, but the way it works.
                return true;
            }
            catch (Exception ex)
            {
                if (ex is Microsoft.SharePoint.Client.ServerException && (ex.Message.IndexOf("Unable to access site") != -1 || ex.Message.IndexOf("Cannot get site") != -1))
                {
                    return false;
                }
                else
                {
                    return true;
                }
            }
        }

        /// <summary>
        /// Checks if a sub site exists
        /// </summary>
        /// <param name="web">Tenant admin web</param>
        /// <param name="siteUrl">URL to the sub site</param>
        /// <returns>True if existing, false if not</returns>
        public static bool SubSiteExistsInTenant(this Web web, string siteUrl)
        {
            try
            {
                return web.CheckIfSiteExistsInTenant(siteUrl, "");
            }
            catch (Exception ex)
            {
                if (ex is Microsoft.SharePoint.Client.ServerException && (ex.Message.IndexOf("Unable to access site") != -1 || ex.Message.IndexOf("Cannot get site") != -1))
                {
                    return true;
                }
                else
                {
                    return false;
                }
            }
        }

        #endregion

        #region Web (site) query, creation and deletion

        /// <summary>
        /// Adds a sub site to an existing site
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="parent">Information about the parent site</param>
        /// <param name="subsite">Information describing the sub site to be added</param>
        /// <param name="inheritPermissions">Does the sub site inherit the permissions of the parent site</param>
        /// <param name="inheritNavigation">Does the sub site inherit the navigation of the parent site</param>
        [Obsolete("Should use CreateWeb(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static void AddSite(this Web web, SiteEntity parent, SiteEntity subsite, bool inheritPermissions, bool inheritNavigation)
        {
            // Call actual implementation
            CreateWeb(web, subsite.Title, subsite.Url, subsite.Description, subsite.Template, (int)subsite.Lcid, inheritPermissions, inheritNavigation);
        }

        /// <summary>
        /// Adds a sub site to an existing site
        /// </summary>
        /// <param name="web">Site to be processed - can be root web or sub site</param>
        /// <param name="title">Title for the site</param>
        /// <param name="description">Description for the new site</param>
        /// <param name="template">Template for the site, like STS#0</param>
        /// <param name="language">Language code for the site, like 1033</param>
        /// <param name="inheritPermissions">Should the new site inherit permissions</param>
        /// <param name="inheritNavigation">Should the new site inherent navigation</param>
        [Obsolete("Should use CreateWeb(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static void AddSite(this Web web, string title, string url, string description, string template, uint language, bool inheritPermissions, bool inheritNavigation)
        {
            // Call centralized route to call internal creation logic
            CreateWeb(web, title, url, description, template, (int)language, inheritPermissions, inheritNavigation);
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="web"></param>
        /// <param name="subsite"></param>
        /// <param name="inheritPermissions"></param>
        /// <param name="inheritNavigation"></param>
        [Obsolete("Should use CreateWeb(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static Web CreateSite(this Web web, SiteEntity subsite, bool inheritPermissions = true, bool inheritNavigation = true)
        {
            // Call actual implementation
            return CreateWeb(web, subsite.Title, subsite.Url, subsite.Description, subsite.Template, (int)subsite.Lcid, inheritPermissions, inheritNavigation);
        }

        [Obsolete("Should use CreateWeb(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static Web CreateSite(this Web web, string title, string url, string description, string template, int language, bool inheritPermissions = true, bool inheritNavigation = true)
        {
            return CreateWeb(web, title, url, description, template, language, inheritPermissions, inheritNavigation);
        }

        /// <summary>
        /// Adds a new child Web (site) to a parent Web.
        /// </summary>
        /// <param name="parentWeb">The parent Web (site) to create under</param>
        /// <param name="subsite">Details of the Web (site) to add. Only Title, Url (as the leaf URL), Description, Template and Language are used.</param>
        /// <param name="inheritPermissions">Specifies whether the new site will inherit permissions from its parent site.</param>
        /// <param name="inheritNavigation">Specifies whether the site inherits navigation.</param>
        /// <returns></returns>
        public static Web CreateWeb(this Web parentWeb, SiteEntity subsite, bool inheritPermissions = true, bool inheritNavigation = true)
        {
            return CreateWeb(parentWeb, subsite.Title, subsite.Url, subsite.Description, subsite.Template, (int)subsite.Lcid, inheritPermissions, inheritNavigation);
        }

        /// <summary>
        /// Adds a new child Web (site) to a parent Web.
        /// </summary>
        /// <param name="parentWeb">The parent Web (site) to create under</param>
        /// <param name="title">The title of the new site. </param>
        /// <param name="leafUrl">A string that represents the URL leaf name.</param>
        /// <param name="description">The description of the new site. </param>
        /// <param name="template">The name of the site template to be used for creating the new site. </param>
        /// <param name="language">The locale ID that specifies the language of the new site. </param>
        /// <param name="inheritPermissions">Specifies whether the new site will inherit permissions from its parent site.</param>
        /// <param name="inheritNavigation">Specifies whether the site inherits navigation.</param>
        public static Web CreateWeb(this Web parentWeb, string title, string leafUrl, string description, string template, int language, bool inheritPermissions = true, bool inheritNavigation = true)
        {
            // TODO: Check for any other illegal characters in SharePoint
            if (leafUrl.Contains('/') || leafUrl.Contains('\\'))
            {
                throw new ArgumentException("The argument must be a single web URL and cannot contain path characters.", "leafUrl");
            }
            LoggingUtility.Internal.TraceInformation((int)EventId.CreateWeb, "Creating web '{0}' with template '{1}'.", leafUrl, template);
            WebCreationInformation creationInfo = new WebCreationInformation()
            {
                Url = leafUrl,
                Title = title,
                Description = description,
                UseSamePermissionsAsParentSite = inheritPermissions,
                WebTemplate = template,
                Language = language
            };

            Web newWeb = parentWeb.Webs.Add(creationInfo);
            newWeb.Navigation.UseShared = inheritNavigation;
            newWeb.Update();

            parentWeb.Context.ExecuteQuery();

            return newWeb;
        }

        /// <summary>
        /// Deletes the child website with the specified leaf URL, from a parent Web, if it exists. 
        /// </summary>
        /// <param name="parentWeb">The parent Web (site) to delete from</param>
        /// <param name="leafUrl">A string that represents the URL leaf name.</param>
        /// <returns>true if the web was deleted; otherwise false if nothing was done</returns>
        public static bool DeleteWeb(this Web parentWeb, string leafUrl)
        {
            // TODO: Check for any other illegal characters in SharePoint
            if (leafUrl.Contains('/') || leafUrl.Contains('\\'))
            {
                throw new ArgumentException("The argument must be a single web URL and cannot contain path characters.", "leafUrl");
            }
            var deleted = false;
            Utility.EnsureWeb(parentWeb.Context, parentWeb, "ServerRelativeUrl");
            var serverRelativeUrl = UrlUtility.Combine(parentWeb.ServerRelativeUrl, leafUrl);
            var webs = parentWeb.Webs;
            // NOTE: Predicate does not take into account a required case-insensitive comparison
            //var results = parentWeb.Context.LoadQuery<Web>(webs.Where(item => item.ServerRelativeUrl == serverRelativeUrl));
            parentWeb.Context.Load(webs, wc => wc.Include(w => w.ServerRelativeUrl));
            parentWeb.Context.ExecuteQuery();
            var existingWeb = webs.FirstOrDefault(item => string.Equals(item.ServerRelativeUrl, serverRelativeUrl, StringComparison.OrdinalIgnoreCase));
            if (existingWeb != null)
            {
                LoggingUtility.Internal.TraceInformation((int)EventId.DeleteWeb, "Deleting web '{0}'.", serverRelativeUrl);
                existingWeb.DeleteObject();
                parentWeb.Context.ExecuteQuery();
                deleted = true;
            }
            else
            {
                LoggingUtility.Internal.TraceVerbose("Delete requested but web '{0}' not found, nothing to do.", serverRelativeUrl);
            }
            return deleted;
        }

        /// <summary>
        /// Gets the collection of the URLs of all Web sites that are contained within the site collection, 
        /// including the top-level site and its subsites.
        /// </summary>
        /// <param name="site">Site collection to retrieve the URLs for.</param>
        /// <returns>An enumeration containing the full URLs as strings.</returns>
        /// <remarks>
        /// <para>
        /// This is analagous to the <code>SPSite.AllWebs</code> property and can be used to get a collection
        /// of all web site URLs to loop through, e.g. for branding.
        /// </para>
        /// </remarks>
        public static IEnumerable<string> GetAllWebUrls(this Site site)
        {
            var siteContext = site.Context;
            siteContext.Load(site, s => s.Url);
            siteContext.ExecuteQuery();
            var queue = new Queue<string>();
            queue.Enqueue(site.Url);
            while (queue.Count > 0)
            {
                var currentUrl = queue.Dequeue();
                using (var webContext = new ClientContext(currentUrl))
                {
                    webContext.Credentials = siteContext.Credentials;
                    webContext.Load(webContext.Web, web => web.Webs);
                    webContext.ExecuteQuery();
                    foreach (var subWeb in webContext.Web.Webs)
                    {
                        queue.Enqueue(subWeb.Url);
                    }
                }
                yield return currentUrl;
            }
        }

        /// <summary>
        /// Checks if a site collection exists
        /// </summary>
        /// <param name="web">Site object opened with credentials that are reused for the site collection check</param>
        /// <param name="siteUrl">Fully qualified URL to the sub site</param>
        /// <returns>true if exists, false otherwise</returns>
        [Obsolete("Should use Context.WebExistsFullUrl(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static bool SiteExists(this Web web, string siteUrl)
        {
            return WebExistsFullUrl(web.Context, siteUrl);
        }

        /// <summary>
        /// Checks if a subsite exists
        /// </summary>
        /// <param name="web">Site object opened with credentials that are reused for the sub site check</param>
        /// <param name="siteUrl">Fully qualified URL to the sub site</param>
        /// <returns>true if exists, false otherwise</returns>
        [Obsolete("Should use Context.WebExists(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static bool SubSiteExists(this Web web, string siteUrl)
        {
            return WebExistsFullUrl(web.Context, siteUrl);
        }

        [Obsolete("Should use WebExists(), to avoid confusion betweeen Site (collection) and Web (site)")]
        [EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        public static bool SubSiteExistsWithUrl(this Web web, string url)
        {
            return WebExists(web, url);
        }

        /// <summary>
        /// Determines if a child Web site with the specified leaf URL exists. 
        /// </summary>
        /// <param name="parentWeb">The Web site to check under</param>
        /// <param name="leafUrl">A string that represents the URL leaf name.</param>
        /// <returns>true if the Web (site) exists; otherwise false</returns>
        public static bool WebExists(this Web parentWeb, string leafUrl)
        {
            // TODO: Check for any other illegal characters in SharePoint
            if (leafUrl.Contains('/') || leafUrl.Contains('\\'))
            {
                throw new ArgumentException("The argument must be a single web URL and cannot contain path characters.", "leafUrl");
            }

            Utility.EnsureWeb(parentWeb.Context, parentWeb, "ServerRelativeUrl");
            var serverRelativeUrl = UrlUtility.Combine(parentWeb.ServerRelativeUrl, leafUrl);
            var webs = parentWeb.Webs;
            // NOTE: Predicate does not take into account a required case-insensitive comparison
            //var results = parentWeb.Context.LoadQuery<Web>(webs.Where(item => item.ServerRelativeUrl == serverRelativeUrl));
            parentWeb.Context.Load(webs, wc => wc.Include(w => w.ServerRelativeUrl));
            parentWeb.Context.ExecuteQuery();
            var exists = webs.Any(item => string.Equals(item.ServerRelativeUrl, serverRelativeUrl, StringComparison.OrdinalIgnoreCase));
            return exists;
        }

        /// <summary>
        /// Determines if a Web (site) exists at the specified full URL, either accessible or that returns an access error.
        /// </summary>
        /// <param name="context">Existing context, used to provide credentials.</param>
        /// <param name="webFullUrl">Full URL of the site to check.</param>
        /// <returns>true if the Web (site) exists; otherwise false</returns>
        public static bool WebExistsFullUrl(this ClientRuntimeContext context, string webFullUrl)
        {
            bool exists = false;
            try
            {
                using(ClientContext testContext = new ClientContext(webFullUrl))
                {
                    testContext.Credentials = context.Credentials;
                    testContext.Load(testContext.Web, w => w.Title);
                    testContext.ExecuteQuery();
                    exists = true;
                }
            }
            catch (Exception ex)
            {
                if (ex is Microsoft.SharePoint.Client.ServerException &&
                    (ex.Message.IndexOf("Unable to access site") != -1 ||
                     ex.Message.IndexOf("Cannot get site") != -1))
                {
                    // Site exists, but you don't have access .. not sure if this is really valid
{'fraction_non_alphanumeric': 0.09861343012704174, 'fraction_numerical': 0.0017713248638838476, 'mean_word_length': 1.8179363390884542, 'pattern_counts': {'":': 0, '<': 503, '<?xml version=': 0, '>': 541, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1566874', 'n_tokens_mistral': 16828, 'n_tokens_neox': 16288, 'n_words': 5058}