<commit_before>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Web;
using MarkdownSharp;
using NuGetGallery.Diagnostics;

namespace NuGetGallery
{
    public class ContentService : IContentService
    {
        // Why not use the ASP.Net Cache?
        // Each entry should _always_ have a value. Values never expire, they just need to be updated. Updates use the existing data,
        // so we don't want data just vanishing from a cache.
        private ConcurrentDictionary<string, ContentItem> _contentCache = new ConcurrentDictionary<string, ContentItem>(StringComparer.OrdinalIgnoreCase);

        private IDiagnosticsSource Trace { get; set; }

        public static readonly string HtmlContentFileExtension = ".html";
        public static readonly string MarkdownContentFileExtension = ".md";

        public IFileStorageService FileStorage { get; protected set; }

        protected ConcurrentDictionary<string, ContentItem> ContentCache { get { return _contentCache; } }

        protected ContentService() {
            Trace = NullDiagnosticsSource.Instance;
        }
        
        public ContentService(IFileStorageService fileStorage, IDiagnosticsService diagnosticsService)
        {
            if (fileStorage == null)
            {
                throw new ArgumentNullException("fileStorage");
            }

            if (diagnosticsService == null)
            {
                throw new ArgumentNullException("diagnosticsService");
            }

            FileStorage = fileStorage;
            Trace = diagnosticsService.GetSource("ContentService");
        }

        public Task<IHtmlString> GetContentItemAsync(string name, TimeSpan expiresIn)
        {
            if (String.IsNullOrEmpty(name))
            {
                throw new ArgumentException(String.Format(CultureInfo.CurrentCulture, Strings.ParameterCannotBeNullOrEmpty, "name"), "name");
            }

            return GetContentItemCore(name, expiresIn);
        }

        // This NNNCore pattern allows arg checking to happen synchronously, before starting the async operation.
        private async Task<IHtmlString> GetContentItemCore(string name, TimeSpan expiresIn)
        {
            using (Trace.Activity("GetContentItem " + name))
            {
                ContentItem cachedItem = null;
                if (ContentCache.TryGetValue(name, out cachedItem) && DateTime.UtcNow < cachedItem.ExpiryUtc)
                {
                    Trace.Verbose("Cache Valid. Expires at: " + item.ExpiryUtc.ToString());
                    return item.Content;
                }
                Trace.Verbose("Cache Expired.");

                // Get the file from the content service
                string htmlFileName = name + HtmlContentFileExtension;
                string markdownFileName = name + MarkdownContentFileExtension;

                foreach (var filename in new[] { htmlFileName, markdownFileName })
                {
                    ContentItem item = await RefreshContentFromFile(filename, cachedItem, expiresIn);
                    if (item != null)
                    {
                        // Cache and return the result
                        Debug.Assert(item.Content != null);
                        ContentCache.AddOrSet(name, item);
                        return item.Content;
                    }
                }

                return new HtmlString(String.Empty);
            }
        }

        private async Task<ContentItem> RefreshContentFromFile(string fileName, ContentItem cachedItem, TimeSpan expiresIn)
        {
            using (Trace.Activity("Downloading Content Item: " + fileName))
            {
                IFileReference reference = await FileStorage.GetFileReferenceAsync(
                    Constants.ContentFolderName,
                    fileName,
                    ifNoneMatch: cachedItem == null ? null : cachedItem.ContentId);

                if (reference == null)
                {
                    Trace.Error("Requested Content File Not Found: " + fileName);
                }
                else
                {
                    // Check the content ID to see if it's different
                    if (item != null && String.Equals(item.ContentId, reference.ContentId, StringComparison.Ordinal))
                    {
                        Trace.Verbose("No change to content item. Using Cache");
                        // No change, just use the cache.
                        result = item.Content;
                    }
                    else
                    {
                        // Process the file
                        Trace.Verbose("Content Item changed. Updating...");
                        using (var stream = reference.OpenRead())
                        {
                            if (stream == null)
                            {
                                Trace.Error("Requested Content File Not Found: " + fileName);
                                reference = null;
                            }
                            else
                            {
                                using (Trace.Activity("Reading Content File: " + fileName))
                                using (var reader = new StreamReader(stream))
                                {
                                    string text = await reader.ReadToEndAsync();
                                    string looseHtml;

                                    if (fileName.EndsWith(".html"))
                                    {
                                        looseHtml = text;
                                    }
                                    else if (fileName.EndsWith(".md"))
                                    {
                                        looseHtml = new Markdown().Transform(text);
                                    }
                                    else
                                    {
                                        looseHtml = "Unknown Content File Type";
                                    }

                                    IHtmlString html = new HtmlString(looseHtml.Trim());

                                    // Prep the new item for the cache
                                    var expiryTime = DateTime.UtcNow + expiresIn;
                                    return new ContentItem(html, expiryTime, reference.ContentId, DateTime.UtcNow);
                                }
                            }
                        }
                    }
                }<commit_msg>Revert bad merge changes in ContentService.cs.<commit_after>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Web;
using MarkdownSharp;
using NuGetGallery.Diagnostics;

namespace NuGetGallery
{
    public class ContentService : IContentService
    {
        // Why not use the ASP.Net Cache?
        // Each entry should _always_ have a value. Values never expire, they just need to be updated. Updates use the existing data,
        // so we don't want data just vanishing from a cache.
        private ConcurrentDictionary<string, ContentItem> _contentCache = new ConcurrentDictionary<string, ContentItem>(StringComparer.OrdinalIgnoreCase);

        private IDiagnosticsSource Trace { get; set; }

        public static readonly string HtmlContentFileExtension = ".html";
        public static readonly string MarkdownContentFileExtension = ".md";

        public IFileStorageService FileStorage { get; protected set; }

        protected ConcurrentDictionary<string, ContentItem> ContentCache { get { return _contentCache; } }

        protected ContentService()
        {
            Trace = NullDiagnosticsSource.Instance;
        }

        public ContentService(IFileStorageService fileStorage, IDiagnosticsService diagnosticsService)
        {
            if (fileStorage == null)
            {
                throw new ArgumentNullException("fileStorage");
            }

            if (diagnosticsService == null)
            {
                throw new ArgumentNullException("diagnosticsService");
            }

            FileStorage = fileStorage;
            Trace = diagnosticsService.GetSource("ContentService");
        }

        public Task<IHtmlString> GetContentItemAsync(string name, TimeSpan expiresIn)
        {
            if (String.IsNullOrEmpty(name))
            {
                throw new ArgumentException(String.Format(CultureInfo.CurrentCulture, Strings.ParameterCannotBeNullOrEmpty, "name"), "name");
            }

            return GetContentItemCore(name, expiresIn);
        }

        // This NNNCore pattern allows arg checking to happen synchronously, before starting the async operation.
        private async Task<IHtmlString> GetContentItemCore(string name, TimeSpan expiresIn)
        {
            using (Trace.Activity("GetContentItem " + name))
            {
                ContentItem cachedItem = null;
                if (ContentCache.TryGetValue(name, out cachedItem) && DateTime.UtcNow < cachedItem.ExpiryUtc)
                {
                    Trace.Verbose("Cache Valid. Expires at: " + cachedItem.ExpiryUtc.ToString());
                    return cachedItem.Content;
                }
                Trace.Verbose("Cache Expired.");

                // Get the file from the content service
                string htmlFileName = name + HtmlContentFileExtension;
                string markdownFileName = name + MarkdownContentFileExtension;

                foreach (var filename in new[] { htmlFileName, markdownFileName })
                {
                    ContentItem item = await RefreshContentFromFile(filename, cachedItem, expiresIn);
                    if (item != null)
                    {
                        // Cache and return the result
                        Debug.Assert(item.Content != null);
                        ContentCache.AddOrSet(name, item);
                        return item.Content;
                    }
                }

                return new HtmlString(String.Empty);
            }
        }

        private async Task<ContentItem> RefreshContentFromFile(string fileName, ContentItem cachedItem, TimeSpan expiresIn)
        {
            using (Trace.Activity("Downloading Content Item: " + fileName))
            {
                IFileReference reference = await FileStorage.GetFileReferenceAsync(
                    Constants.ContentFolderName,
                    fileName,
                    ifNoneMatch: cachedItem == null ? null : cachedItem.ContentId);

                if (reference == null)
                {
                    Trace.Error("Requested Content File Not Found: " + fileName);
                    return null;
                }

                // Check the content ID to see if it's different
                if (cachedItem != null && String.Equals(cachedItem.ContentId, reference.ContentId, StringComparison.Ordinal))
                {
                    Trace.Verbose("No change to content item. Using Cache");

                    // Update the expiry time
                    cachedItem.ExpiryUtc = DateTime.UtcNow + expiresIn;
                    Trace.Verbose(String.Format("Updating Cache: {0} expires at {1}", fileName, cachedItem.ExpiryUtc));
                    return cachedItem;
                }

                // Retrieve the content
                Trace.Verbose("Content Item changed. Trying to update...");
                try
                {
                    using (var stream = reference.OpenRead())
                    {
                        if (stream == null)
                        {
                            Trace.Error("Requested Content File Not Found: " + fileName);
                            return null;
                        }
                        else
                        {
                            using (Trace.Activity("Reading Content File: " + fileName))
                            {
                                using (var reader = new StreamReader(stream))
                                {
                                    string text = await reader.ReadToEndAsync();
                                    string looseHtml;

                                    if (fileName.EndsWith(".html"))
                                    {
                                        looseHtml = text;
                                    }
                                    else if (fileName.EndsWith(".md"))
                                    {
                                        looseHtml = new Markdown().Transform(text);
                                    }
                                    else
                                    {
                                        looseHtml = "Unknown Content File Type";
                                    }

                                    IHtmlString html = new HtmlString(looseHtml.Trim());

                                    // Prep the new item for the cache
                                    var expiryTime = DateTime.UtcNow + expiresIn;
                                    return new ContentItem(html, expiryTime, reference.ContentId, DateTime.UtcNow);
                                }
                            }
                        }
                    }
                }
{'fraction_non_alphanumeric': 0.0689856767027185, 'fraction_numerical': 0.00014615609470914936, 'mean_word_length': 1.270238885202389, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5440591', 'n_tokens_mistral': 2995, 'n_tokens_neox': 2797, 'n_words': 763}