<commit_before>﻿using System;
using System.Collections.Generic;
using System.Configuration;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Xml.Linq;
using CollectionsOnline.Core.Indexes;
using NLog;
using Raven.Abstractions.Data;
using Raven.Client;
using Constants = CollectionsOnline.Core.Config.Constants;

namespace CollectionsOnline.Import.Imports
{
    public class SitemapImport : IImport
    {
        private readonly Logger _log = LogManager.GetCurrentClassLogger();
        private readonly IDocumentStore _documentStore;

        public SitemapImport(IDocumentStore documentStore)
        {
            _documentStore = documentStore;
        }

        public void Run()
        {            
            _log.Debug("Beginning sitemap creation");

            var count = 0;

            var sitemapNodes = new List<SitemapNode>();
            var sitemapNodeIndexes = new List<List<SitemapNode>>();
            var skip = 0;
            
            while (true)
            {
                using (var documentSession = _documentStore.OpenSession())
                {
                    var query = documentSession.Advanced
                        .DocumentQuery<CombinedIndexResult, CombinedIndex>()
                        .SelectFields<SitemapResult>()
                        .Skip(skip)
                        .Take(Constants.PagingStreamSize);

                    QueryHeaderInformation queryHeaderInformation;
                    var enumerator = documentSession.Advanced.Stream(query, out queryHeaderInformation);

                    // Create all sitemap nodes
                    while (enumerator.MoveNext())
                    {
                        if (ImportCanceled())
                            return;

                        sitemapNodes.Add(new SitemapNode(new Uri(string.Format("{0}{1}",
                            ConfigurationManager.AppSettings["CanonicalSiteBase"], enumerator.Current.Document.Id)),
                            enumerator.Current.Document.DateModified));

                        count++;

                        // Add sitemap to urlset once we reach our maximum url limit and create a new list to hold the next batch
                        if (count % Constants.MaxSitemapUrls == 0)
                        {
                            sitemapNodeIndexes.Add(sitemapNodes);
                            sitemapNodes = new List<SitemapNode>();
                        }
                    }

                    _log.Trace("Sitemap caching progress... {0}/{1}", count, queryHeaderInformation.TotalResults);

                    skip += Constants.PagingStreamSize;

                    if (count == queryHeaderInformation.TotalResults)
                        break;
                }
            }

            // Add the last sitemapNodes to our index
            if(sitemapNodes.Any())
                sitemapNodeIndexes.Add(sitemapNodes);

            // Create destination folder
            var directoryInfo = new DirectoryInfo(string.Format("{0}\\sitemaps", ConfigurationManager.AppSettings["WebSitePath"]));
            if (!directoryInfo.Exists)
                directoryInfo.Create();

            // Create then save sitemaps as xml
            count = 0;
            XNamespace xmlns = "http://www.sitemaps.org/schemas/sitemap/0.9";
            var utf8WithoutBom = new System.Text.UTF8Encoding(false);
            var sitemapIndex = new XElement(xmlns + "sitemapindex");

            foreach (var sitemapNodeIndex in sitemapNodeIndexes)
            {
                count++;

                sitemapIndex.Add(new XElement(xmlns + "sitemap",
                    new XElement(xmlns + "loc", Uri.EscapeUriString(string.Format("{0}sitemap-set-{1}.xml", ConfigurationManager.AppSettings["CanonicalSiteBase"], count))),
                    new XElement(xmlns + "lastmod", sitemapNodeIndex.OrderByDescending(x => x.LastModified).Select(x => x.LastModified).First().ToString("yyyy-MM-ddTHH:mm:sszzz", CultureInfo.InvariantCulture))));

                var sitemapUrlset = new XElement(xmlns + "urlset");
                foreach (var sitemapNode in sitemapNodeIndex)
                {
                    var sitemapUrl = new XElement(xmlns + "url",
                        new XElement(xmlns + "loc", Uri.EscapeUriString(sitemapNode.Url.AbsoluteUri)),
                        new XElement(xmlns + "lastmod", sitemapNode.LastModified.ToString("yyyy-MM-ddTHH:mm:sszzz", CultureInfo.InvariantCulture)),
                        new XElement(xmlns + "changefreq", sitemapNode.Frequency.ToString().ToLowerInvariant()));

                    if (sitemapNode.Priority.HasValue)
                    {
                        sitemapUrl.Add(new XElement(xmlns + "priority", sitemapNode.Priority.ToString().ToLowerInvariant()));
                    }

                    sitemapUrlset.Add(sitemapUrl);
                }

                // Save sitemap set with gzip compression
                using (var file = File.Create(string.Format("{0}\\sitemaps\\sitemap-set-{1}.xml.gz", ConfigurationManager.AppSettings["WebSitePath"], count)))
                using (var gzip = new GZipStream(file, CompressionMode.Compress, false))
                {
                    sitemapUrlset.Save(gzip);
                }
            }

            // Save sitemap index<commit_msg>fix sitemap import<commit_after>﻿using System;
using System.Collections.Generic;
using System.Configuration;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Xml.Linq;
using CollectionsOnline.Core.Indexes;
using NLog;
using Raven.Abstractions.Data;
using Raven.Client;
using Constants = CollectionsOnline.Core.Config.Constants;

namespace CollectionsOnline.Import.Imports
{
    public class SitemapImport : IImport
    {
        private readonly Logger _log = LogManager.GetCurrentClassLogger();
        private readonly IDocumentStore _documentStore;

        public SitemapImport(IDocumentStore documentStore)
        {
            _documentStore = documentStore;
        }

        public void Run()
        {            
            _log.Debug("Beginning sitemap creation");

            var count = 0;

            var sitemapNodes = new List<SitemapNode>();
            var sitemapNodeIndexes = new List<List<SitemapNode>>();
            var skip = 0;
            
            while (true)
            {
                using (var documentSession = _documentStore.OpenSession())
                {
                    var query = documentSession.Advanced
                        .DocumentQuery<CombinedIndexResult, CombinedIndex>()
                        .SelectFields<SitemapResult>()
                        .Skip(skip)
                        .Take(Constants.PagingStreamSize);

                    QueryHeaderInformation queryHeaderInformation;
                    var enumerator = documentSession.Advanced.Stream(query, out queryHeaderInformation);

                    // Create all sitemap nodes
                    while (enumerator.MoveNext())
                    {
                        if (ImportCanceled())
                            return;

                        sitemapNodes.Add(new SitemapNode(new Uri(string.Format("{0}{1}",
                            ConfigurationManager.AppSettings["CanonicalSiteBase"], enumerator.Current.Document.Id)),
                            enumerator.Current.Document.DateModified));

                        count++;

                        // Add sitemap to urlset once we reach our maximum url limit and create a new list to hold the next batch
                        if (count % Constants.MaxSitemapUrls == 0)
                        {
                            sitemapNodeIndexes.Add(sitemapNodes);
                            sitemapNodes = new List<SitemapNode>();
                        }
                    }

                    _log.Trace("Sitemap caching progress... {0}/{1}", count, queryHeaderInformation.TotalResults);

                    skip += Constants.PagingStreamSize;

                    if (count == queryHeaderInformation.TotalResults)
                        break;
                }
            }

            // Add the last sitemapNodes to our index
            if(sitemapNodes.Any())
                sitemapNodeIndexes.Add(sitemapNodes);

            // Create destination folder
            var directoryInfo = new DirectoryInfo(string.Format("{0}\\sitemaps", ConfigurationManager.AppSettings["WebSitePath"]));
            if (!directoryInfo.Exists)
                directoryInfo.Create();

            // Create then save sitemaps as xml
            count = 0;
            XNamespace xmlns = "http://www.sitemaps.org/schemas/sitemap/0.9";
            var utf8WithoutBom = new System.Text.UTF8Encoding(false);
            var sitemapIndex = new XElement(xmlns + "sitemapindex");

            foreach (var sitemapNodeIndex in sitemapNodeIndexes)
            {
                count++;

                sitemapIndex.Add(new XElement(xmlns + "sitemap",
                    new XElement(xmlns + "loc", Uri.EscapeUriString(string.Format("{0}sitemap-set-{1}.xml.gz", ConfigurationManager.AppSettings["CanonicalSiteBase"], count))),
                    new XElement(xmlns + "lastmod", sitemapNodeIndex.OrderByDescending(x => x.LastModified).Select(x => x.LastModified).First().ToString("yyyy-MM-ddTHH:mm:sszzz", CultureInfo.InvariantCulture))));

                var sitemapUrlset = new XElement(xmlns + "urlset");
                foreach (var sitemapNode in sitemapNodeIndex)
                {
                    var sitemapUrl = new XElement(xmlns + "url",
                        new XElement(xmlns + "loc", Uri.EscapeUriString(sitemapNode.Url.AbsoluteUri)),
                        new XElement(xmlns + "lastmod", sitemapNode.LastModified.ToString("yyyy-MM-ddTHH:mm:sszzz", CultureInfo.InvariantCulture)),
                        new XElement(xmlns + "changefreq", sitemapNode.Frequency.ToString().ToLowerInvariant()));

                    if (sitemapNode.Priority.HasValue)
                    {
                        sitemapUrl.Add(new XElement(xmlns + "priority", sitemapNode.Priority.ToString().ToLowerInvariant()));
                    }

                    sitemapUrlset.Add(sitemapUrl);
                }

                // Save sitemap set with gzip compression
                using (var file = File.Create(string.Format("{0}\\sitemaps\\sitemap-set-{1}.xml.gz", ConfigurationManager.AppSettings["WebSitePath"], count)))
                using (var gzip = new GZipStream(file, CompressionMode.Compress, false))
                {
                    sitemapUrlset.Save(gzip);
                }
            }

            // Save sitemap index
{'fraction_non_alphanumeric': 0.09305555555555556, 'fraction_numerical': 0.003148148148148148, 'mean_word_length': 1.7151835093011563, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3517255', 'n_tokens_mistral': 2819, 'n_tokens_neox': 2632, 'n_words': 531}