<commit_before>#region License, Terms and Author(s)
//
// Schnell - Wiki widgets
// Copyright (c) 2007 <NAME>. All rights reserved.
//
//  Author(s):
//      <NAME>, http://www.raboof.com
//
// This library is free software; you can redistribute it and/or modify it 
// under the terms of the GNU Lesser General Public License as published by 
// the Free Software Foundation; either version 2.1 of the License, or (at 
// your option) any later version.
//
// This library is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 
// License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with this library; if not, write to the Free Software Foundation, 
// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 
//
#endregion

namespace Schnell
{
    #region Imports

    using System;

    #endregion

    [ Serializable ]
    public abstract class WikiToken {}

    [ Serializable ]
    public sealed class WikiEndToken : WikiToken
    {
        private readonly WikiToken _start;
        
        public WikiEndToken(WikiToken start)
        {
            _start = start;
        }
        
        public WikiToken Start
        {
            get { return _start; }
        }
    }

    //
    // Inline tokens
    //

    [ Serializable ] public sealed class WikiBoldToken : WikiToken { }
    [ Serializable ] public sealed class WikiItalicToken : WikiToken { }
    [ Serializable ] public sealed class WikiStrikeToken : WikiToken { }
    [ Serializable ] public sealed class WikiSuperscriptToken : WikiToken { }
    [ Serializable ] public sealed class WikiSubscriptToken : WikiToken { }
    [ Serializable ] public sealed class WikiMonospaceToken : WikiToken { }

    //
    // Block tokens
    //

    [ Serializable ] public sealed class WikiCodeToken : WikiToken { }
    [ Serializable ] public sealed class WikiQuoteToken : WikiToken { }
    [ Serializable ] public sealed class WikiParaToken : WikiToken { }
    [ Serializable ] public sealed class WikiNumberedListToken : WikiToken { }
    [ Serializable ] public sealed class WikiBulletedListToken : WikiToken { }
    [ Serializable ] public sealed class WikiListItemToken : WikiToken { }
    [ Serializable ] public sealed class WikiTableToken : WikiToken { }
    [ Serializable ] public sealed class WikiRowToken : WikiToken { }
    [ Serializable ] public sealed class WikiCellToken : WikiToken { }
    
    [ Serializable ] 
    public sealed class WikiHeadingToken : WikiToken
    {
        private readonly int _level;
        
        public WikiHeadingToken(int level)
        {
            if (level < 1 || level > 6)
                throw new ArgumentOutOfRangeException("level", level, null);

            _level = level;
        }
        
        public int Level
        {
            get { return _level; }
        }
    }

    [ Serializable ]
    public sealed class WikiTagToken : WikiToken
    {
        private readonly string _key;
        private readonly string _value;
        
        public WikiTagToken(string key, string value)
        {
            _key = key;
            _value = value;
        }
        
        public string Key
        {
            get { return _key ?? string.Empty; }
        }

        public string Value
        {
            get { return _value ?? string.Empty; }
        }
    }

    [ Serializable ]
    public sealed class WikiImageToken : WikiToken
    {
        private readonly string _src;
        private readonly string _href;
        
        public WikiImageToken(string src) :
            this(src, null) {}

        public WikiImageToken(string src, string href)
        {
            _src = src;
            _href = href;
        }

        public string Src
        {
            get { return _src ?? string.Empty; }
        }
        
        public string Href
        {
            get { return _href ?? string.Empty; }
        }
    }

    [ Serializable ]
    public sealed class WikiHyperlinkToken : WikiToken
    {
        private readonly string _href;
        private readonly string _text;

        public WikiHyperlinkToken(string href) : 
            this(href, null) {}

        public WikiHyperlinkToken(string href, string text)
        {
            _href = href;
            _text = string.IsNullOrEmpty(text) ? href : text;
        }
        
        public string Href
        {
            get { return _href ?? string.Empty; }
        }

        public string Text
        {
            get { return _text ?? string.Empty; }
        }
    }

    [ Serializable ]
    public sealed class WikiWordToken : WikiToken
    {
        private readonly string _word;

        public WikiWordToken(string word)
        {
            _word = word;
        }

        public string Word
        {
            get { return _word ?? string.Empty; }
        }
    }
    
    [ Serializable ]
    public sealed class WikiTextToken : WikiToken
    {
        private readonly string _text;
        
        public WikiTextToken(string text)
        {
            _text = text;
        }
        
        public string Text
        {
            get { return _text ?? string.Empty; }
        }
    }
}<commit_msg>Updated all WikiToken subclasses to handle mask null strings during construction rather than during property get-access since all tokens are immutable.<commit_after>#region License, Terms and Author(s)
//
// Schnell - Wiki widgets
// Copyright (c) 2007 <NAME>. All rights reserved.
//
//  Author(s):
//      <NAME>, http://www.raboof.com
//
// This library is free software; you can redistribute it and/or modify it 
// under the terms of the GNU Lesser General Public License as published by 
// the Free Software Foundation; either version 2.1 of the License, or (at 
// your option) any later version.
//
// This library is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 
// License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with this library; if not, write to the Free Software Foundation, 
// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 
//
#endregion

namespace Schnell
{
    #region Imports

    using System;

    #endregion

    [ Serializable ]
    public abstract class WikiToken {}

    [ Serializable ]
    public sealed class WikiEndToken : WikiToken
    {
        private readonly WikiToken _start;
        
        public WikiEndToken(WikiToken start)
        {
            _start = start;
        }
        
        public WikiToken Start
        {
            get { return _start; }
        }
    }

    //
    // Inline tokens
    //

    [ Serializable ] public sealed class WikiBoldToken : WikiToken { }
    [ Serializable ] public sealed class WikiItalicToken : WikiToken { }
    [ Serializable ] public sealed class WikiStrikeToken : WikiToken { }
    [ Serializable ] public sealed class WikiSuperscriptToken : WikiToken { }
    [ Serializable ] public sealed class WikiSubscriptToken : WikiToken { }
    [ Serializable ] public sealed class WikiMonospaceToken : WikiToken { }

    //
    // Block tokens
    //

    [ Serializable ] public sealed class WikiCodeToken : WikiToken { }
    [ Serializable ] public sealed class WikiQuoteToken : WikiToken { }
    [ Serializable ] public sealed class WikiParaToken : WikiToken { }
    [ Serializable ] public sealed class WikiNumberedListToken : WikiToken { }
    [ Serializable ] public sealed class WikiBulletedListToken : WikiToken { }
    [ Serializable ] public sealed class WikiListItemToken : WikiToken { }
    [ Serializable ] public sealed class WikiTableToken : WikiToken { }
    [ Serializable ] public sealed class WikiRowToken : WikiToken { }
    [ Serializable ] public sealed class WikiCellToken : WikiToken { }
    
    [ Serializable ] 
    public sealed class WikiHeadingToken : WikiToken
    {
        private readonly int _level;
        
        public WikiHeadingToken(int level)
        {
            if (level < 1 || level > 6)
                throw new ArgumentOutOfRangeException("level", level, null);

            _level = level;
        }
        
        public int Level
        {
            get { return _level; }
        }
    }

    [ Serializable ]
    public sealed class WikiTagToken : WikiToken
    {
        private readonly string _key;
        private readonly string _value;
        
        public WikiTagToken(string key, string value)
        {
            _key = key ?? string.Empty;
            _value = value ?? string.Empty;
        }
        
        public string Key
        {
            get { return _key; }
        }

        public string Value
        {
            get { return _value; }
        }
    }

    [ Serializable ]
    public sealed class WikiImageToken : WikiToken
    {
        private readonly string _src;
        private readonly string _href;
        
        public WikiImageToken(string src) :
            this(src, null) {}

        public WikiImageToken(string src, string href)
        {
            _src = src ?? string.Empty;
            _href = href ?? string.Empty;
        }

        public string Src
        {
            get { return _src; }
        }
        
        public string Href
        {
            get { return _href; }
        }
    }

    [ Serializable ]
    public sealed class WikiHyperlinkToken : WikiToken
    {
        private readonly string _href;
        private readonly string _text;

        public WikiHyperlinkToken(string href) : 
            this(href, null) {}

        public WikiHyperlinkToken(string href, string text)
        {
            _href = href ?? string.Empty;
            _text = string.IsNullOrEmpty(text) ? (href ?? string.Empty) : text;
        }
        
        public string Href
        {
            get { return _href; }
        }

        public string Text
        {
            get { return _text; }
        }
    }

    [ Serializable ]
    public sealed class WikiWordToken : WikiToken
    {
        private readonly string _word;

        public WikiWordToken(string word)
        {
            _word = word ?? string.Empty;
        }

        public string Word
        {
            get { return _word; }
        }
    }
    
    [ Serializable ]
    public sealed class WikiTextToken : WikiToken
    {
        private readonly string _text;
        
        public WikiTextToken(string text)
        {
            _text = text ?? string.Empty;
        }
        
        public string Text
        {
            get { return _text; }
        }
    }
}
{'fraction_non_alphanumeric': 0.07186179011211175, 'fraction_numerical': 0.0040433743797096125, 'mean_word_length': 1.8201606633842964, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1274863', 'n_tokens_mistral': 2953, 'n_tokens_neox': 2748, 'n_words': 940}