<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Threading.Tasks;
using System.Xml.Linq;
using Newtonsoft.Json.Linq;
using NuGet.Packaging;
using NuGet.Packaging.Core;
using NuGet.Versioning;

namespace Sleet
{
    public class Catalog : ISleetService, IPackagesLookup, IRootIndex
    {
        private readonly SleetContext _context;

        public string Name { get; } = "Catalog";

        public string RootIndex { get; } = "catalog/index.json";

        public Catalog(SleetContext context)
        {
            _context = context;
        }

        public ISleetFile RootIndexFile
        {
            get
            {
                return _context.Source.Get(RootIndex);
            }
        }

        /// <summary>
        /// Add a package to the catalog.
        /// </summary>
        public async Task AddPackage(PackageInput packageInput)
        {
            // Create package details page
            var packageDetails = CreatePackageDetails(packageInput);
            var packageDetailsFile = _context.Source.Get(UriUtility.CreateUri(packageDetails["@id"].ToString()));
            await packageDetailsFile.Write(packageDetails, _context.Log, _context.Token);

            // Add catalog page entry
            var catalogIndexUri = _context.Source.GetPath("/catalog/index.json");
            var catalogIndexFile = _context.Source.Get(catalogIndexUri);
            var catalogIndexJson = await catalogIndexFile.GetJson(_context.Log, _context.Token);

            catalogIndexJson["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            catalogIndexJson["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();

            var pages = GetItems(catalogIndexJson);

            var currentPageUri = GetCurrentPage(catalogIndexJson);
            var currentPageFile = _context.Source.Get(currentPageUri);

            var pageCommits = new List<JObject>();

            if (await currentPageFile.Exists(_context.Log, _context.Token))
            {
                var currentPageJson = await currentPageFile.GetJson(_context.Log, _context.Token);

                pageCommits = GetItems(currentPageJson);
            }
            else
            {
                var newPage = JsonUtility.Create(currentPageFile.EntityUri, "CatalogPage");
                newPage["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
                newPage["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
                newPage["count"] = 0;

                newPage = JsonLDTokenComparer.Format(newPage);

                var pageArray = (JArray)catalogIndexJson["items"];
                pageArray.Add(newPage);

                // Update pages
                pages = GetItems(catalogIndexJson);
            }

            // Create commit
            var pageCommit = JsonUtility.Create(packageDetailsFile.EntityUri, "nuget:PackageDetails");
            pageCommit["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            pageCommit["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
            pageCommit["nuget:id"] = packageInput.Identity.Id;
            pageCommit["nuget:version"] = packageInput.Identity.Version.ToFullVersionString();
            pageCommit["sleet:operation"] = "add";

            pageCommits.Add(pageCommit);

            // Write catalog page
            var pageJson = CreateCatalogPage(catalogIndexUri, currentPageUri, pageCommits);

            await currentPageFile.Write(pageJson, _context.Log, _context.Token);

            // Update index
            var pageEntry = pages.Where(e => e["@id"].ToString() == currentPageFile.EntityUri.AbsoluteUri).Single();
            pageEntry["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            pageEntry["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
            pageEntry["count"] = pageCommits.Count;

            catalogIndexJson["count"] = pages.Count;
            catalogIndexJson["nuget:lastCreated"] = DateTimeOffset.UtcNow.GetDateString();

            catalogIndexJson = JsonLDTokenComparer.Format(catalogIndexJson);

            await catalogIndexFile.Write(catalogIndexJson, _context.Log, _context.Token);
        }

        public async Task RemovePackage(PackageIdentity package)
        {
            // Create package details page
            var packageDetails = CreateDeleteDetails(package, string.Empty);
            var packageDetailsFile = _context.Source.Get(packageDetails.GetEntityId());
            await packageDetailsFile.Write(packageDetails, _context.Log, _context.Token);

            // Add catalog page entry
            var catalogIndexUri = _context.Source.GetPath("/catalog/index.json");
            var catalogIndexFile = _context.Source.Get(catalogIndexUri);
            var catalogIndexJson = await catalogIndexFile.GetJson(_context.Log, _context.Token);

            catalogIndexJson["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            catalogIndexJson["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();

            var pages = GetItems(catalogIndexJson);

            var currentPageUri = GetCurrentPage(catalogIndexJson);
            var currentPageFile = _context.Source.Get(currentPageUri);

            var pageCommits = new List<JObject>();

            if (await currentPageFile.Exists(_context.Log, _context.Token))
            {
                var currentPageJson = await currentPageFile.GetJson(_context.Log, _context.Token);

                pageCommits = GetItems(currentPageJson);
            }
            else
            {
                var newPage = JsonUtility.Create(currentPageFile.EntityUri, "CatalogPage");
                newPage["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
                newPage["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
                newPage["count"] = 0;

                newPage = JsonLDTokenComparer.Format(newPage);

                var pageArray = (JArray)catalogIndexJson["items"];
                pageArray.Add(newPage);

                // Update pages
                pages = GetItems(catalogIndexJson);
            }

            // Create commit
            var pageCommit = JsonUtility.Create(packageDetailsFile.EntityUri, "nuget:PackageDetails");
            pageCommit["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            pageCommit["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
            pageCommit["nuget:id"] = package.Id;
            pageCommit["nuget:version"] = package.Version.ToFullVersionString();
            pageCommit["sleet:operation"] = "remove";

            pageCommits.Add(pageCommit);

            // Write catalog page
            var pageJson = CreateCatalogPage(catalogIndexUri, currentPageUri, pageCommits);

            await currentPageFile.Write(pageJson, _context.Log, _context.Token);

            // Update index
            var pageEntry = pages.Where(e => e["@id"].ToString() == currentPageFile.EntityUri.AbsoluteUri).Single();
            pageEntry["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            pageEntry["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
            pageEntry["count"] = pageCommits.Count;

            catalogIndexJson["count"] = pages.Count;
            catalogIndexJson["nuget:lastDeleted"] = DateTimeOffset.UtcNow.GetDateString();

            catalogIndexJson = JsonLDTokenComparer.Format(catalogIndexJson);

            await catalogIndexFile.Write(catalogIndexJson, _context.Log, _context.Token);
        }

        /// <summary>
        /// Catalog index page.
        /// </summary><commit_msg>Changing commit type for deletes to nuget:PackageDelete<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Threading.Tasks;
using System.Xml.Linq;
using Newtonsoft.Json.Linq;
using NuGet.Packaging;
using NuGet.Packaging.Core;
using NuGet.Versioning;

namespace Sleet
{
    public class Catalog : ISleetService, IPackagesLookup, IRootIndex
    {
        private readonly SleetContext _context;

        public string Name { get; } = "Catalog";

        public string RootIndex { get; } = "catalog/index.json";

        public Catalog(SleetContext context)
        {
            _context = context;
        }

        public ISleetFile RootIndexFile
        {
            get
            {
                return _context.Source.Get(RootIndex);
            }
        }

        /// <summary>
        /// Add a package to the catalog.
        /// </summary>
        public async Task AddPackage(PackageInput packageInput)
        {
            // Create package details page
            var packageDetails = CreatePackageDetails(packageInput);
            var packageDetailsFile = _context.Source.Get(UriUtility.CreateUri(packageDetails["@id"].ToString()));
            await packageDetailsFile.Write(packageDetails, _context.Log, _context.Token);

            // Add catalog page entry
            var catalogIndexUri = _context.Source.GetPath("/catalog/index.json");
            var catalogIndexFile = _context.Source.Get(catalogIndexUri);
            var catalogIndexJson = await catalogIndexFile.GetJson(_context.Log, _context.Token);

            catalogIndexJson["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            catalogIndexJson["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();

            var pages = GetItems(catalogIndexJson);

            var currentPageUri = GetCurrentPage(catalogIndexJson);
            var currentPageFile = _context.Source.Get(currentPageUri);

            var pageCommits = new List<JObject>();

            if (await currentPageFile.Exists(_context.Log, _context.Token))
            {
                var currentPageJson = await currentPageFile.GetJson(_context.Log, _context.Token);

                pageCommits = GetItems(currentPageJson);
            }
            else
            {
                var newPage = JsonUtility.Create(currentPageFile.EntityUri, "CatalogPage");
                newPage["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
                newPage["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
                newPage["count"] = 0;

                newPage = JsonLDTokenComparer.Format(newPage);

                var pageArray = (JArray)catalogIndexJson["items"];
                pageArray.Add(newPage);

                // Update pages
                pages = GetItems(catalogIndexJson);
            }

            // Create commit
            var pageCommit = JsonUtility.Create(packageDetailsFile.EntityUri, "nuget:PackageDetails");
            pageCommit["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            pageCommit["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
            pageCommit["nuget:id"] = packageInput.Identity.Id;
            pageCommit["nuget:version"] = packageInput.Identity.Version.ToFullVersionString();
            pageCommit["sleet:operation"] = "add";

            pageCommits.Add(pageCommit);

            // Write catalog page
            var pageJson = CreateCatalogPage(catalogIndexUri, currentPageUri, pageCommits);

            await currentPageFile.Write(pageJson, _context.Log, _context.Token);

            // Update index
            var pageEntry = pages.Where(e => e["@id"].ToString() == currentPageFile.EntityUri.AbsoluteUri).Single();
            pageEntry["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            pageEntry["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
            pageEntry["count"] = pageCommits.Count;

            catalogIndexJson["count"] = pages.Count;
            catalogIndexJson["nuget:lastCreated"] = DateTimeOffset.UtcNow.GetDateString();

            catalogIndexJson = JsonLDTokenComparer.Format(catalogIndexJson);

            await catalogIndexFile.Write(catalogIndexJson, _context.Log, _context.Token);
        }

        public async Task RemovePackage(PackageIdentity package)
        {
            // Create package details page
            var packageDetails = CreateDeleteDetails(package, string.Empty);
            var packageDetailsFile = _context.Source.Get(packageDetails.GetEntityId());
            await packageDetailsFile.Write(packageDetails, _context.Log, _context.Token);

            // Add catalog page entry
            var catalogIndexUri = _context.Source.GetPath("/catalog/index.json");
            var catalogIndexFile = _context.Source.Get(catalogIndexUri);
            var catalogIndexJson = await catalogIndexFile.GetJson(_context.Log, _context.Token);

            catalogIndexJson["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            catalogIndexJson["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();

            var pages = GetItems(catalogIndexJson);

            var currentPageUri = GetCurrentPage(catalogIndexJson);
            var currentPageFile = _context.Source.Get(currentPageUri);

            var pageCommits = new List<JObject>();

            if (await currentPageFile.Exists(_context.Log, _context.Token))
            {
                var currentPageJson = await currentPageFile.GetJson(_context.Log, _context.Token);

                pageCommits = GetItems(currentPageJson);
            }
            else
            {
                var newPage = JsonUtility.Create(currentPageFile.EntityUri, "CatalogPage");
                newPage["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
                newPage["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
                newPage["count"] = 0;

                newPage = JsonLDTokenComparer.Format(newPage);

                var pageArray = (JArray)catalogIndexJson["items"];
                pageArray.Add(newPage);

                // Update pages
                pages = GetItems(catalogIndexJson);
            }

            // Create commit
            var pageCommit = JsonUtility.Create(packageDetailsFile.EntityUri, "nuget:PackageDelete");
            pageCommit["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            pageCommit["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
            pageCommit["nuget:id"] = package.Id;
            pageCommit["nuget:version"] = package.Version.ToFullVersionString();
            pageCommit["sleet:operation"] = "remove";

            pageCommits.Add(pageCommit);

            // Write catalog page
            var pageJson = CreateCatalogPage(catalogIndexUri, currentPageUri, pageCommits);

            await currentPageFile.Write(pageJson, _context.Log, _context.Token);

            // Update index
            var pageEntry = pages.Where(e => e["@id"].ToString() == currentPageFile.EntityUri.AbsoluteUri).Single();
            pageEntry["commitId"] = _context.CommitId.ToString().ToLowerInvariant();
            pageEntry["commitTimeStamp"] = DateTimeOffset.UtcNow.GetDateString();
            pageEntry["count"] = pageCommits.Count;

            catalogIndexJson["count"] = pages.Count;
            catalogIndexJson["nuget:lastDeleted"] = DateTimeOffset.UtcNow.GetDateString();

            catalogIndexJson = JsonLDTokenComparer.Format(catalogIndexJson);

            await catalogIndexFile.Write(catalogIndexJson, _context.Log, _context.Token);
        }

        /// <summary>
        /// Catalog index page.
        /// </summary>
{'fraction_non_alphanumeric': 0.1037572809319593, 'fraction_numerical': 0.00025603277219484095, 'mean_word_length': 2.4482454204369897, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3077005', 'n_tokens_mistral': 4000, 'n_tokens_neox': 3945, 'n_words': 683}