<commit_before>﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using Mechanical.Conditions;
using Mechanical.Core;
using Mechanical.IO;
using Mechanical.MagicBag;

namespace Mechanical.DataStores
{
    /// <summary>
    /// Encapsulates generic, data store related functionality.
    /// </summary>
    public static class DataStore
    {
        //// NOTE: Data store names have strict restrictions
        ////         - they may not be null
        ////         - they may not be empty
        ////         - they may only contain english letters of either case, digits, or underscores
        ////         - they may not start with a digit
        ////         - they are considered case-sensitive
        ////         - they may be no longer than 255 characters

        //// NOTE: In contrast, data store paths have a few more restrictions:
        ////         - all valid data store names are valid data store paths
        ////         - a path separator may only occur between valid data store names (so paths may not start or end with them, and they may not be next to each other)
        ////       but are also more forgiving:
        ////         - empty strings are valid paths as well (they represent the parent "directory" of the root node)
        ////         - there is no length restriction

        #region Private Fields

        private const int MaxNameLength = 255 - 1; // '\0' --> -1
        private const int MaxPathLength = 260 - 3 - 1; // "c:\" --> -3
        private const char EscapeCharacter = '_';

        private static readonly char[] ValidFirstCharacters;

        #endregion

        #region Static Constructor

        static DataStore()
        {
            ValidFirstCharacters = GetValidFirstCharacters();
        }

        #endregion

        #region IsValidName

        private static char[] GetValidFirstCharacters()
        {
            var chars = new List<char>();
            for( char ch = 'a'; ch <= 'z'; ++ch )
                chars.Add(ch);
            for( char ch = 'A'; ch <= 'Z'; ++ch )
                chars.Add(ch);
            chars.Add('_');
            return chars.ToArray();
        }

        /// <summary>
        /// Determines whether the specified string is a valid data store name.
        /// </summary>
        /// <param name="name">The string to examine.</param>
        /// <returns><c>true</c> if the specified string is a valid data store name; otherwise, <c>false</c>.</returns>
        public static bool IsValidName( Substring name )
        {
            if( name.NullOrEmpty )
                return false;

            if( name.Length > MaxNameLength ) // max. NTFS file name length
                return false;

            if( !IsValidFirstCharacter(name[0]) )
                return false;

            for( int i = 1; i < name.Length; ++i )
            {
                if( !IsValidMiddleCharacter(name[i]) )
                    return false;
            }

            return true;
        }

#if !MECHANICAL_NET4
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
        private static bool IsValidFirstCharacter( char ch )
        {
            return ('a' <= ch && ch <= 'z')
                || ('A' <= ch && ch <= 'Z')
                || ch == '_';
        }

#if !MECHANICAL_NET4
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
        private static bool IsValidMiddleCharacter( char ch )
        {
            return IsValidFirstCharacter(ch)
                || ('0' <= ch && ch <= '9');
        }

        #endregion

        #region GenerateName

        private static readonly Random Rnd = new Random();

        /// <summary>
        /// Generates a data store name. The generated name may still not be valid
        /// if it is added to a data store object, with a child of the same name.
        /// The generated name may be completely different than the one provided,
        /// therefore it should be saved, if restoring the original name is desired.
        /// </summary>
        /// <param name="from">The string to generate the name from deterministically; or <c>null</c> to generate a random hash based name.</param>
        /// <returns>The data store name generated.</returns>
        public static string GenerateName( string from = null )
        {
            if( !from.NullOrEmpty() )
            {
                // always try to escape, even if it would be data store compatible (think underscores!)
                var escaped = EscapeName(from);
                if( DataStore.IsValidName(escaped) )
                    return escaped;

                // escape didn't work, generate deterministic (case-sensitive) hash
                byte[] bytes = DefaultEncoding.GetBytes(from);
                var sb = new StringBuilder(32);
#if !SILVERLIGHT
                bytes = System.Security.Cryptography.MD5.Create().ComputeHash(bytes); // MD5 is fine, since we don't worry about cryptographic security, and it's shorter than SHA1 and others
                foreach( var b in bytes )
                    sb.Append(b.ToString("x2"));
#else
                sb.Append(Convert.ToBase64String(bytes), startIndex: 0, count: 32);
#endif
                if( !IsValidFirstCharacter(sb[0]) )
                    sb[0] = '_'; // does not decrease hash deviation (unlike a letter would)
                return sb.ToString();
            }

            // generate random name
            var guid = Guid.NewGuid().ToString("N");
            if( !IsValidFirstCharacter(guid[0]) )
                guid = ValidFirstCharacters[Rnd.Next(ValidFirstCharacters.Length)] + guid.Substring(startIndex: 1);
            return guid;
        }

        #endregion

        #region EscapeName, UnescapeName, EscapePath, UnescapePath

#if !MECHANICAL_NET4
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
        private static void InitializeEscapeBuffer( ref StringBuilder sb, ref bool stringBuilderUsedBefore, Substring str )
        {
            if( !stringBuilderUsedBefore )
            {
                stringBuilderUsedBefore = true;<commit_msg>Work on random data store name and hash generation.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using Mechanical.Conditions;
using Mechanical.Core;
using Mechanical.IO;
using Mechanical.MagicBag;

namespace Mechanical.DataStores
{
    /// <summary>
    /// Encapsulates generic, data store related functionality.
    /// </summary>
    public static class DataStore
    {
        //// NOTE: Data store names have strict restrictions
        ////         - they may not be null
        ////         - they may not be empty
        ////         - they may only contain english letters of either case, digits, or underscores
        ////         - they may not start with a digit
        ////         - they are considered case-sensitive
        ////         - they may be no longer than 255 characters

        //// NOTE: In contrast, data store paths have a few more restrictions:
        ////         - all valid data store names are valid data store paths
        ////         - a path separator may only occur between valid data store names (so paths may not start or end with them, and they may not be next to each other)
        ////       but are also more forgiving:
        ////         - empty strings are valid paths as well (they represent the parent "directory" of the root node)
        ////         - there is no length restriction

        #region Private Fields

        private const int MaxNameLength = 255 - 1; // '\0' --> -1
        private const int MaxPathLength = 260 - 3 - 1; // "c:\" --> -3
        private const char EscapeCharacter = '_';

        private static readonly char[] ValidFirstCharacters;

        #endregion

        #region Static Constructor

        static DataStore()
        {
            ValidFirstCharacters = GetValidFirstCharacters();
        }

        #endregion

        #region IsValidName

        private static char[] GetValidFirstCharacters()
        {
            var chars = new List<char>();
            for( char ch = 'a'; ch <= 'z'; ++ch )
                chars.Add(ch);
            for( char ch = 'A'; ch <= 'Z'; ++ch )
                chars.Add(ch);
            chars.Add('_');
            return chars.ToArray();
        }

        /// <summary>
        /// Determines whether the specified string is a valid data store name.
        /// </summary>
        /// <param name="name">The string to examine.</param>
        /// <returns><c>true</c> if the specified string is a valid data store name; otherwise, <c>false</c>.</returns>
        public static bool IsValidName( Substring name )
        {
            if( name.NullOrEmpty )
                return false;

            if( name.Length > MaxNameLength ) // max. NTFS file name length
                return false;

            if( !IsValidFirstCharacter(name[0]) )
                return false;

            for( int i = 1; i < name.Length; ++i )
            {
                if( !IsValidMiddleCharacter(name[i]) )
                    return false;
            }

            return true;
        }

#if !MECHANICAL_NET4
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
        private static bool IsValidFirstCharacter( char ch )
        {
            return ('a' <= ch && ch <= 'z')
                || ('A' <= ch && ch <= 'Z')
                || ch == '_';
        }

#if !MECHANICAL_NET4
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
        private static bool IsValidMiddleCharacter( char ch )
        {
            return IsValidFirstCharacter(ch)
                || ('0' <= ch && ch <= '9');
        }

        #endregion

        #region GenerateHash, GenerateNames

        private static readonly Random Rnd = new Random();

        /// <summary>
        /// Generates a data store compatible hash string. The generated name may still not be valid
        /// if it is added to a data store object, with a child of the same name.
        /// </summary>
        /// <param name="from">The string to generate the name from deterministically; or <c>null</c> to generate a random hash based name.</param>
        /// <returns>The data store name generated.</returns>
        public static string GenerateHash( string from = null )
        {
            if( !from.NullOrEmpty() )
            {
                // generate deterministic (case-sensitive) hash
                byte[] bytes = DefaultEncoding.GetBytes(from);
                var sb = new StringBuilder(32);
#if !SILVERLIGHT
                bytes = System.Security.Cryptography.MD5.Create().ComputeHash(bytes); // MD5 is fine, since we don't worry about cryptographic security, and it's shorter than SHA1 and others
                foreach( var b in bytes )
                    sb.Append(b.ToString("x2"));
#else
                sb.Append(Convert.ToBase64String(bytes), startIndex: 0, count: 32);
#endif
                if( !IsValidFirstCharacter(sb[0]) )
                    sb[0] = '_'; // does not decrease hash deviation (unlike a letter would)
                return sb.ToString();
            }
            else
            {
                // generate random hash
                var guid = Guid.NewGuid().ToString("N");
                if( !IsValidFirstCharacter(guid[0]) )
                    guid = ValidFirstCharacters[Rnd.Next(ValidFirstCharacters.Length)] + guid.Substring(startIndex: 1);
                return guid;
            }
        }

        /// <summary>
        /// Generates unique data store names, starting with the shortest
        /// possible lengths.
        /// </summary>
        /// <returns>The data store names generated.</returns>
        public static IEnumerable<string> GenerateNames()
        {
            var sb = new StringBuilder();
            var digits = new List<IEnumerator<char>>();
            while( true )
            {
                MoveNext(digits);
                if( digits.Count > MaxNameLength )
                    yield break;

                sb.Clear();
                for( int i = 0; i < digits.Count; ++i )
                    sb.Append(digits[i].Current);

                yield return sb.ToString();
            }
        }

        private static IEnumerable<char> GenerateUniqueCharacters( bool firstCharacter )
        {
            char ch;

            for( ch = 'a'; ch <= 'z'; ++ch )
                yield return ch;

            for( ch = 'A'; ch <= 'Z'; ++ch )
                yield return ch;

            if( !firstCharacter )
            {
                for( ch = '0'; ch <= '9'; ++ch )
                    yield return ch;
            }

            yield return '_';
        }

        private static void MoveNext( List<IEnumerator<char>> digits )
        {
            // increment "digits"
            for( int i = digits.Count - 1; i >= 0; --i )
            {
                if( digits[i].MoveNext() )
                {
                    // "digit" increased, nothing to "carry over"
                    return;
                }
                else
                {
                    // no more characters for current "digit":
                    // reset it, and let the previous one be increased
                    digits[i] = GenerateUniqueCharacters(firstCharacter: false).GetEnumerator();
                    digits[i].MoveNext();
                }
            }

            // all "digits" reset: we need an extra digit at the front
            digits.Insert(0, GenerateUniqueCharacters(firstCharacter: true).GetEnumerator());
            digits[0].MoveNext();
        }

        /// <summary>
        /// Generates data store names. Each item (starting with the second),
        /// appends a unique suffix to the end.
        /// </summary>
        /// <param name="from">The string to generate the names from.</param>
        /// <returns>The data store names generated.</returns>
        public static IEnumerable<string> GenerateNames( string from )
        {
            if( from.NullOrEmpty() )
                return GenerateNames();

            from = RemoveSpecialCharacters(from);
            if( from.NullOrEmpty() )
                return GenerateNames(from);

            return GenerateNamesUsing(from);
        }

        private static string RemoveSpecialCharacters( string str )
        {
            //// NOTE: string.Normalize might come in handy here,
            ////       but Silverlight and WinRT do no currently support it

            // do nothing if there is nothing to work with
            if( str.NullOrEmpty() )
                return str;

            // do nothing if already valid...
            str = str.Trim(); // ... except for leading and trailing white space
            if( DataStore.IsValidName(str) )
                return str;

#if !SILVERLIGHT
            // remove diacritics
            str = string.Join(string.Empty, str.Normalize(NormalizationForm.FormD).Where(c => char.GetUnicodeCategory(c) != UnicodeCategory.NonSpacingMark));
#else
            // an alternative, that works surprisingly well (though probably not perfectly)
            // works nicely with this string: "éáűőúöüóí ÉÁŰŐÚÖÜÓÍ ıİçÇğĞşŞ"
            str = Encoding.ASCII.GetString(Encoding.GetEncoding("iso-8859-8").GetBytes(str));;
#endif
            if( str.NullOrEmpty()
             || DataStore.IsValidName(str) )
                return str;

            // remove unsupported characters
            // (replaces ' ' with '_')
            var chars = new List<char>(str.ToCharArray());
            while( chars.Count > 0
                && !IsValidFirstCharacter(chars[0]) )
            {
                if( chars[0] == ' ' )
                {
                    chars[0] = '_';
                    break;
                }
                else
                    chars.RemoveAt(0);
            }
            for( int i = 1; i < chars.Count; )
            {
                if( IsValidMiddleCharacter(chars[i]) )
                {
                    ++i;
                }
                else
                {
                    if( chars[i] == ' ' )
                    {
                        chars[i] = '_';
                        ++i;
                    }
                    else
                        chars.RemoveAt(i);
                }
            }
            return new string(chars.ToArray());
        }

        private static IEnumerable<string> GenerateNamesUsing( string dataStoreName )
        {
            yield return dataStoreName;

            int length, newLength;
            foreach( var suffix in GenerateNames() )
            {
                length = dataStoreName.Length + 1 + suffix.Length;
                if( length > MaxNameLength )
                {
                    newLength = dataStoreName.Length - (length - MaxNameLength);
                    if( newLength <= 0 )
                        yield return suffix; // return suffixes only from this point on
                    else
                        yield return dataStoreName.Substring(0, newLength) + '_' + suffix;
                }

                yield return dataStoreName + '_' + suffix;
            }
        }

        #endregion

        #region EscapeName, UnescapeName, EscapePath, UnescapePath

#if !MECHANICAL_NET4
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
        private static void InitializeEscapeBuffer( ref StringBuilder sb, ref bool stringBuilderUsedBefore, Substring str )
        {
            if( !stringBuilderUsedBefore )
            {
                stringBuilderUsedBefore = true;
{'fraction_non_alphanumeric': 0.09997760859829825, 'fraction_numerical': 0.005317957904164801, 'mean_word_length': 1.5466856735566643, 'pattern_counts': {'":': 1, '<': 88, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3005933', 'n_tokens_mistral': 4650, 'n_tokens_neox': 4325, 'n_words': 1439}