<commit_before>﻿
// debugging
// define WRTCONS to write cleaning details & steps to console
// leave it wrapped within #if DEBUG to make sure it does leak
// into RELEASE, see http://issues.umbraco.org/issue/U4-4199
#if DEBUG
#undef WRTCONS
#endif

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
using Umbraco.Core.Configuration;

namespace Umbraco.Core.Strings
{
    /// <summary>
    /// New default implementation of string functions for short strings such as aliases or url segments.
    /// </summary>
    /// <remarks>
    /// <para>Not optimized to work on large bodies of text.</para>
    /// <para>Meant to replace <c>LegacyShortStringHelper</c> where/when backward compatibility is not an issue.</para>
    /// <para>NOTE: pre-filters run _before_ the string is re-encoded.</para>
    /// </remarks>
    public class DefaultShortStringHelper : IShortStringHelper
    {
        #region Ctor and vars

        public DefaultShortStringHelper()
        {
            InitializeLegacyUrlReplaceCharacters();
        }

        /// <summary>
        /// Freezes the helper so it can prevents its configuration from being modified.
        /// </summary>
        /// <remarks>Will be called by <c>ShortStringHelperResolver</c> when resolution freezes.</remarks>
        public void Freeze()
        {
            _frozen = true;
        }

        // see notes for CleanAsciiString
        //// beware! the order is quite important here!
        //const string ValidStringCharactersSource = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
        //readonly static char[] ValidStringCharacters;

        private CultureInfo _defaultCulture = CultureInfo.InvariantCulture;
        private bool _frozen;
        private readonly Dictionary<CultureInfo, Dictionary<CleanStringType, Config>> _configs = new Dictionary<CultureInfo, Dictionary<CleanStringType, Config>>();

        // see notes for CleanAsciiString
        //static DefaultShortStringHelper()
        //{
        //    ValidStringCharacters = ValidStringCharactersSource.ToCharArray();
        //}

        #endregion

        #region Filters

        private readonly Dictionary<string, string> _urlReplaceCharacters = new Dictionary<string, string>();

        private void InitializeLegacyUrlReplaceCharacters()
        {
            foreach (var node in UmbracoConfig.For.UmbracoSettings().RequestHandler.CharCollection)
            {
                if (node.Char.IsNullOrWhiteSpace() == false)
                    _urlReplaceCharacters[node.Char] = node.Replacement;
            }
        }

        /// <summary>
        /// Returns a new string in which characters have been replaced according to the Umbraco settings UrlReplaceCharacters.
        /// </summary>
        /// <param name="s">The string to filter.</param>
        /// <returns>The filtered string.</returns>
        public string ApplyUrlReplaceCharacters(string s)
        {
            return s.ReplaceMany(_urlReplaceCharacters);
        }

        // ok to be static here because it's not configureable in any way
        private static readonly char[] InvalidFileNameChars =
            Path.GetInvalidFileNameChars()
            .Union("!*'();:@&=+$,/?%#[]-~{}\"<>\\^`| ".ToCharArray())
            .Distinct()
            .ToArray();

        public static bool IsValidFileNameChar(char c)
        {
            return InvalidFileNameChars.Contains(c) == false;
        }

        public static string CutMaxLength(string text, int length)
        {
            return text.Length <= length ? text : text.Substring(0, length);
        }

        #endregion

        #region Configuration

        private void EnsureNotFrozen()
        {
            if (_frozen)
                throw new InvalidOperationException("Cannot configure the helper once it is frozen.");            
        }

        /// <summary>
        /// Sets a default culture.
        /// </summary>
        /// <param name="culture">The default culture.</param>
        /// <returns>The short string helper.</returns>
        public DefaultShortStringHelper WithDefaultCulture(CultureInfo culture)
        {
            EnsureNotFrozen();
            _defaultCulture = culture;
            return this;
        }

        public DefaultShortStringHelper WithConfig(Config config)
        {
            return WithConfig(_defaultCulture, CleanStringType.RoleMask, config);
        }

        public DefaultShortStringHelper WithConfig(CleanStringType stringRole, Config config)
        {
            return WithConfig(_defaultCulture, stringRole, config);
        }

        public DefaultShortStringHelper WithConfig(CultureInfo culture, CleanStringType stringRole, Config config)
        {
            if (config == null)
                throw new ArgumentNullException("config");

            EnsureNotFrozen();
            if (_configs.ContainsKey(culture) == false)
                _configs[culture] = new Dictionary<CleanStringType, Config>();
            _configs[culture][stringRole] = config.Clone(); // clone so it can't be changed
            return this;
        }

        /// <summary>
        /// Sets the default configuration.
        /// </summary>
        /// <returns>The short string helper.</returns>
        public DefaultShortStringHelper WithDefaultConfig()
        {
            return WithConfig(CleanStringType.UrlSegment, new Config
            {
                PreFilter = ApplyUrlReplaceCharacters,
                PostFilter = x => CutMaxLength(x, 240),
                IsTerm = (c, leading) => char.IsLetterOrDigit(c) || c == '_', // letter, digit or underscore
                StringType = (UmbracoConfig.For.UmbracoSettings().RequestHandler.ConvertUrlsToAscii ? CleanStringType.Ascii : CleanStringType.Utf8) | CleanStringType.LowerCase,
                BreakTermsOnUpper = false,
                Separator = '-'
            }).WithConfig(CleanStringType.FileName, new Config
            {
                PreFilter = ApplyUrlReplaceCharacters,
                IsTerm = (c, leading) => char.IsLetterOrDigit(c) || c == '_', // letter, digit or underscore
                StringType = CleanStringType.Utf8 | CleanStringType.LowerCase,
                BreakTermsOnUpper = false,
                Separator = '-'
            }).WithConfig(CleanStringType.Alias, new Config
            {
                PreFilter = ApplyUrlReplaceCharacters,
                IsTerm = (c, leading) => leading 
                    ? char.IsLetter(c) // only letters
                    : (char.IsLetterOrDigit(c) || c == '_'), // letter, digit or underscore
                StringType = CleanStringType.Ascii | CleanStringType.UmbracoCase,
                BreakTermsOnUpper = false
            }).WithConfig(CleanStringType.ConvertCase, new Config<commit_msg>Allow leading underscore in aliases<commit_after>﻿
// debugging
// define WRTCONS to write cleaning details & steps to console
// leave it wrapped within #if DEBUG to make sure it does leak
// into RELEASE, see http://issues.umbraco.org/issue/U4-4199
#if DEBUG
#undef WRTCONS
#endif

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
using Umbraco.Core.Configuration;

namespace Umbraco.Core.Strings
{
    /// <summary>
    /// New default implementation of string functions for short strings such as aliases or url segments.
    /// </summary>
    /// <remarks>
    /// <para>Not optimized to work on large bodies of text.</para>
    /// <para>Meant to replace <c>LegacyShortStringHelper</c> where/when backward compatibility is not an issue.</para>
    /// <para>NOTE: pre-filters run _before_ the string is re-encoded.</para>
    /// </remarks>
    public class DefaultShortStringHelper : IShortStringHelper
    {
        #region Ctor and vars

        public DefaultShortStringHelper()
        {
            InitializeLegacyUrlReplaceCharacters();
        }

        /// <summary>
        /// Freezes the helper so it can prevents its configuration from being modified.
        /// </summary>
        /// <remarks>Will be called by <c>ShortStringHelperResolver</c> when resolution freezes.</remarks>
        public void Freeze()
        {
            _frozen = true;
        }

        // see notes for CleanAsciiString
        //// beware! the order is quite important here!
        //const string ValidStringCharactersSource = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
        //readonly static char[] ValidStringCharacters;

        private CultureInfo _defaultCulture = CultureInfo.InvariantCulture;
        private bool _frozen;
        private readonly Dictionary<CultureInfo, Dictionary<CleanStringType, Config>> _configs = new Dictionary<CultureInfo, Dictionary<CleanStringType, Config>>();

        // see notes for CleanAsciiString
        //static DefaultShortStringHelper()
        //{
        //    ValidStringCharacters = ValidStringCharactersSource.ToCharArray();
        //}

        #endregion

        #region Filters

        private readonly Dictionary<string, string> _urlReplaceCharacters = new Dictionary<string, string>();

        private void InitializeLegacyUrlReplaceCharacters()
        {
            foreach (var node in UmbracoConfig.For.UmbracoSettings().RequestHandler.CharCollection)
            {
                if (node.Char.IsNullOrWhiteSpace() == false)
                    _urlReplaceCharacters[node.Char] = node.Replacement;
            }
        }

        /// <summary>
        /// Returns a new string in which characters have been replaced according to the Umbraco settings UrlReplaceCharacters.
        /// </summary>
        /// <param name="s">The string to filter.</param>
        /// <returns>The filtered string.</returns>
        public string ApplyUrlReplaceCharacters(string s)
        {
            return s.ReplaceMany(_urlReplaceCharacters);
        }

        // ok to be static here because it's not configureable in any way
        private static readonly char[] InvalidFileNameChars =
            Path.GetInvalidFileNameChars()
            .Union("!*'();:@&=+$,/?%#[]-~{}\"<>\\^`| ".ToCharArray())
            .Distinct()
            .ToArray();

        public static bool IsValidFileNameChar(char c)
        {
            return InvalidFileNameChars.Contains(c) == false;
        }

        public static string CutMaxLength(string text, int length)
        {
            return text.Length <= length ? text : text.Substring(0, length);
        }

        #endregion

        #region Configuration

        private void EnsureNotFrozen()
        {
            if (_frozen)
                throw new InvalidOperationException("Cannot configure the helper once it is frozen.");            
        }

        /// <summary>
        /// Sets a default culture.
        /// </summary>
        /// <param name="culture">The default culture.</param>
        /// <returns>The short string helper.</returns>
        public DefaultShortStringHelper WithDefaultCulture(CultureInfo culture)
        {
            EnsureNotFrozen();
            _defaultCulture = culture;
            return this;
        }

        public DefaultShortStringHelper WithConfig(Config config)
        {
            return WithConfig(_defaultCulture, CleanStringType.RoleMask, config);
        }

        public DefaultShortStringHelper WithConfig(CleanStringType stringRole, Config config)
        {
            return WithConfig(_defaultCulture, stringRole, config);
        }

        public DefaultShortStringHelper WithConfig(CultureInfo culture, CleanStringType stringRole, Config config)
        {
            if (config == null)
                throw new ArgumentNullException("config");

            EnsureNotFrozen();
            if (_configs.ContainsKey(culture) == false)
                _configs[culture] = new Dictionary<CleanStringType, Config>();
            _configs[culture][stringRole] = config.Clone(); // clone so it can't be changed
            return this;
        }

        /// <summary>
        /// Sets the default configuration.
        /// </summary>
        /// <returns>The short string helper.</returns>
        public DefaultShortStringHelper WithDefaultConfig()
        {
            return WithConfig(CleanStringType.UrlSegment, new Config
            {
                PreFilter = ApplyUrlReplaceCharacters,
                PostFilter = x => CutMaxLength(x, 240),
                IsTerm = (c, leading) => char.IsLetterOrDigit(c) || c == '_', // letter, digit or underscore
                StringType = (UmbracoConfig.For.UmbracoSettings().RequestHandler.ConvertUrlsToAscii ? CleanStringType.Ascii : CleanStringType.Utf8) | CleanStringType.LowerCase,
                BreakTermsOnUpper = false,
                Separator = '-'
            }).WithConfig(CleanStringType.FileName, new Config
            {
                PreFilter = ApplyUrlReplaceCharacters,
                IsTerm = (c, leading) => char.IsLetterOrDigit(c) || c == '_', // letter, digit or underscore
                StringType = CleanStringType.Utf8 | CleanStringType.LowerCase,
                BreakTermsOnUpper = false,
                Separator = '-'
            }).WithConfig(CleanStringType.Alias, new Config
            {
                PreFilter = ApplyUrlReplaceCharacters,
                IsTerm = (c, leading) => leading
                    ? char.IsLetter(c) || c == '_' // only letters or underscore - but underscore can be used only in doctypealias
                    : (char.IsLetterOrDigit(c) || c == '_'), // letter, digit or underscore
                StringType = CleanStringType.Ascii | CleanStringType.UmbracoCase,
                BreakTermsOnUpper = false
            }).WithConfig(CleanStringType.ConvertCase, new Config
{'fraction_non_alphanumeric': 0.09692164845368188, 'fraction_numerical': 0.0029997857295907433, 'mean_word_length': 2.1721794290892613, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2490538', 'n_tokens_mistral': 3605, 'n_tokens_neox': 3456, 'n_words': 928}