<commit_before>﻿using System;
using System.Drawing;
using System.Drawing.Drawing2D;
using System.Drawing.Imaging;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using DD4T.ContentModel;
using DD4T.ContentModel.Contracts.Caching;
using DD4T.ContentModel.Factories;
using DD4T.Factories;
using DD4T.Factories.Caching;
using Sdl.Web.Common.Configuration;
using Sdl.Web.Common.Logging;
using Sdl.Web.Mvc.Configuration;
using Sdl.Web.Mvc.Statics;
using DD4T.ContentModel.Exceptions;

namespace Sdl.Web.DD4T.Statics
{
    /// <summary>
    /// Ensures a Binary file is cached on the file-system from the Tridion Broker DB
    /// </summary>
    public class BinaryFileManager : BaseStaticFileManager, IBinaryFileManager
    {
        #region caching
        private ICacheAgent _cacheAgent;
        /// <summary>
        /// Get or set the CacheAgent
        /// </summary>  
        public ICacheAgent CacheAgent 
        {
            get
            {
                return _cacheAgent ?? (_cacheAgent = new DefaultCacheAgent());
            }
            set
            {
                _cacheAgent = value;
            }
        }
        public const string CacheKeyFormatBinary = "Binary_{0}";
        private static string GetCacheKey(string url)
        {
            return string.Format(CacheKeyFormatBinary, url);
        }
        #endregion

        #region IBinaryFileManager
        /// <summary>
        /// Main worker method reads binary from Broker and stores it in file-system
        /// </summary>
        /// <returns></returns>
        public bool ProcessRequest(HttpRequest request)
        {
            string urlPath = request.Url.AbsolutePath.Replace("/" + SiteConfiguration.GetLocalStaticsUrl(WebRequestContext.Localization.LocalizationId), String.Empty);
            string physicalPath = request.PhysicalPath;
            Log.Debug("Start processing " + urlPath);
            return ProcessUrl(urlPath, false, physicalPath);
        }

        public string GetStaticContent(string urlPath, Localization loc, bool cacheSinceLastRefresh = false)
        {
            if (ProcessUrl(urlPath, cacheSinceLastRefresh, null, loc))
            {
                var filePath = GetFilePathFromUrl(urlPath, loc);
                if (File.Exists(filePath))
                {
                    return Encoding.UTF8.GetString(File.ReadAllBytes(filePath));
                }
            }
            return null;
        }

        /// <summary>
        /// Main worker method reads binary from Broker and stores it in file-system
        /// </summary>
        /// <returns></returns>
        public bool ProcessUrl(string urlPath, bool cacheSinceLastRefresh = false, string physicalPath = null, Localization loc = null)
        {
            Dimensions dimensions;
            if (loc==null)
            {
                loc = WebRequestContext.Localization;
            }
            bool connectionError = false;
            if (physicalPath == null)
            {
                physicalPath = GetFilePathFromUrl(urlPath, loc);
            }
            urlPath = StripDimensions(urlPath, out dimensions);
            string cacheKey = GetCacheKey(urlPath);
            DateTime? lastPublishedDate = CacheAgent.Load(cacheKey) as DateTime?;
            if (lastPublishedDate == null)
            {
                BinaryFactory.BinaryProvider.PublicationId = Int32.Parse(loc.LocalizationId);
                try
                {
                    DateTime lpb = BinaryFactory.FindLastPublishedDate(urlPath);
                    if (lpb != DateTime.MinValue.AddSeconds(1)) // this is the secret code for 'does not exist'
                    {
                        lastPublishedDate = lpb;
                        CacheAgent.Store(cacheKey, "Binary", lastPublishedDate);
                    }
                }
                catch (NullReferenceException)
                {
                    //Binary not found, this should return a min date, but theres a bug in DD4T where it throws a NRE
                    //DO NOTHING - binary removed later
                }
                catch (Exception ex)
                {
                    Log.Error(ex);
                    connectionError = true; 
                }
            }
            if (connectionError)
            {
                if (File.Exists(physicalPath))
                {
                    //if theres an error connecting, but we still have a version on disk, serve this
                    return true;
                }
                return false;
            }
            if (lastPublishedDate != null)
            {
                if (File.Exists(physicalPath))
                {
                    if (cacheSinceLastRefresh && loc.LastSettingsRefresh.CompareTo(lastPublishedDate) < 0)
                    {
                        //File has been modified since last application start but we don't care
                        Log.Debug("Binary {0} is modified, but only since last application restart, so no action required", urlPath);
                        return true;
                    }
                    FileInfo fi = new FileInfo(physicalPath);
                    if (fi.Length > 0)
                    {
                        DateTime fileModifiedDate = File.GetLastWriteTime(physicalPath);
                        if (fileModifiedDate.CompareTo(lastPublishedDate) >= 0)
                        {
                            Log.Debug("Binary {0} is still up to date, no action required", urlPath);
                            return true;
                        }
                    }
                }
            }

            // the normal situation (where a binary is still in Tridion and it is present on the file system already and it is up to date) is now covered
            // Let's handle the exception situations. 
            IBinary binary = GetBinaryFromBroker(urlPath,loc);
            if (binary==null)
            {
                Log.Debug("Binary with url {0} not found", urlPath);
                // binary does not exist in Tridion, it should be removed from the local file system too<commit_msg>Updated logging to add publication id when binary not found<commit_after>﻿using System;
using System.Drawing;
using System.Drawing.Drawing2D;
using System.Drawing.Imaging;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using DD4T.ContentModel;
using DD4T.ContentModel.Contracts.Caching;
using DD4T.ContentModel.Factories;
using DD4T.Factories;
using DD4T.Factories.Caching;
using Sdl.Web.Common.Configuration;
using Sdl.Web.Common.Logging;
using Sdl.Web.Mvc.Configuration;
using Sdl.Web.Mvc.Statics;
using DD4T.ContentModel.Exceptions;

namespace Sdl.Web.DD4T.Statics
{
    /// <summary>
    /// Ensures a Binary file is cached on the file-system from the Tridion Broker DB
    /// </summary>
    public class BinaryFileManager : BaseStaticFileManager, IBinaryFileManager
    {
        #region caching
        private ICacheAgent _cacheAgent;
        /// <summary>
        /// Get or set the CacheAgent
        /// </summary>  
        public ICacheAgent CacheAgent 
        {
            get
            {
                return _cacheAgent ?? (_cacheAgent = new DefaultCacheAgent());
            }
            set
            {
                _cacheAgent = value;
            }
        }
        public const string CacheKeyFormatBinary = "Binary_{0}";
        private static string GetCacheKey(string url)
        {
            return string.Format(CacheKeyFormatBinary, url);
        }
        #endregion

        #region IBinaryFileManager
        /// <summary>
        /// Main worker method reads binary from Broker and stores it in file-system
        /// </summary>
        /// <returns></returns>
        public bool ProcessRequest(HttpRequest request)
        {
            string urlPath = request.Url.AbsolutePath.Replace("/" + SiteConfiguration.GetLocalStaticsUrl(WebRequestContext.Localization.LocalizationId), String.Empty);
            string physicalPath = request.PhysicalPath;
            Log.Debug("Start processing " + urlPath);
            return ProcessUrl(urlPath, false, physicalPath);
        }

        public string GetStaticContent(string urlPath, Localization loc, bool cacheSinceLastRefresh = false)
        {
            if (ProcessUrl(urlPath, cacheSinceLastRefresh, null, loc))
            {
                var filePath = GetFilePathFromUrl(urlPath, loc);
                if (File.Exists(filePath))
                {
                    return Encoding.UTF8.GetString(File.ReadAllBytes(filePath));
                }
            }
            return null;
        }

        /// <summary>
        /// Main worker method reads binary from Broker and stores it in file-system
        /// </summary>
        /// <returns></returns>
        public bool ProcessUrl(string urlPath, bool cacheSinceLastRefresh = false, string physicalPath = null, Localization loc = null)
        {
            Dimensions dimensions;
            if (loc==null)
            {
                loc = WebRequestContext.Localization;
            }
            bool connectionError = false;
            if (physicalPath == null)
            {
                physicalPath = GetFilePathFromUrl(urlPath, loc);
            }
            urlPath = StripDimensions(urlPath, out dimensions);
            string cacheKey = GetCacheKey(urlPath);
            DateTime? lastPublishedDate = CacheAgent.Load(cacheKey) as DateTime?;
            if (lastPublishedDate == null)
            {
                BinaryFactory.BinaryProvider.PublicationId = Int32.Parse(loc.LocalizationId);
                try
                {
                    DateTime lpb = BinaryFactory.FindLastPublishedDate(urlPath);
                    if (lpb != DateTime.MinValue.AddSeconds(1)) // this is the secret code for 'does not exist'
                    {
                        lastPublishedDate = lpb;
                        CacheAgent.Store(cacheKey, "Binary", lastPublishedDate);
                    }
                }
                catch (NullReferenceException)
                {
                    //Binary not found, this should return a min date, but theres a bug in DD4T where it throws a NRE
                    //DO NOTHING - binary removed later
                }
                catch (Exception ex)
                {
                    Log.Error(ex);
                    connectionError = true; 
                }
            }
            if (connectionError)
            {
                if (File.Exists(physicalPath))
                {
                    //if theres an error connecting, but we still have a version on disk, serve this
                    return true;
                }
                return false;
            }
            if (lastPublishedDate != null)
            {
                if (File.Exists(physicalPath))
                {
                    if (cacheSinceLastRefresh && loc.LastSettingsRefresh.CompareTo(lastPublishedDate) < 0)
                    {
                        //File has been modified since last application start but we don't care
                        Log.Debug("Binary {0} is modified, but only since last application restart, so no action required", urlPath);
                        return true;
                    }
                    FileInfo fi = new FileInfo(physicalPath);
                    if (fi.Length > 0)
                    {
                        DateTime fileModifiedDate = File.GetLastWriteTime(physicalPath);
                        if (fileModifiedDate.CompareTo(lastPublishedDate) >= 0)
                        {
                            Log.Debug("Binary {0} is still up to date, no action required", urlPath);
                            return true;
                        }
                    }
                }
            }

            // the normal situation (where a binary is still in Tridion and it is present on the file system already and it is up to date) is now covered
            // Let's handle the exception situations. 
            IBinary binary = GetBinaryFromBroker(urlPath,loc);
            if (binary==null)
            {
                Log.Debug("Binary with url {0} not found in publication {1}", urlPath, loc.LocalizationId);
                // binary does not exist in Tridion, it should be removed from the local file system too
{'fraction_non_alphanumeric': 0.07744510978043913, 'fraction_numerical': 0.0032734530938123754, 'mean_word_length': 1.5188015282525638, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '356190', 'n_tokens_mistral': 3092, 'n_tokens_neox': 2909, 'n_words': 854}