<commit_before>﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Linq;
using System.Globalization;

namespace NeoSmart.Unicode
{
    public struct Codepoint : IComparable<Codepoint>, IComparable<UInt32>, IEquatable<Codepoint>, 
        IEquatable<string>, IComparable<string>, IEquatable<char>
    {
        public readonly UInt32 Value;

        public Codepoint(UInt32 value)
        {
            Value = value;
        }

        public Codepoint(long value)
            : this ((UInt32) value)
        { }

        /// <summary>
        /// Create a unicode codepoint from hexadecimal representation, supporting U+xxxx and 0xYYYY notation.
        /// </summary>
        /// <param name="hexValue"></param>
        public Codepoint(string hexValue)
        {
            if ((hexValue.StartsWith("0x") || hexValue.StartsWith("U+") || hexValue.StartsWith("u+")))
            {
                hexValue = hexValue.Substring(2);
            }
            if (!uint.TryParse(hexValue, NumberStyles.HexNumber, CultureInfo.CurrentCulture.NumberFormat, out Value))
            {
                throw new UnsupportedCodepointException();
            }
        }

        public UInt32 AsUtf32 => Value;

        //https://en.wikipedia.org/wiki/UTF-16
        public UInt16[] AsUtf16
        {
           get
            {
                //U+0000 to U+D7FF and U+E000 to U+FFFF
                if (Value <= 0xFFFF)
                {
                    return new UInt16[1] { (UInt16)Value };
                }

                //U+10000 to U+10FFFF
                if (Value >= 0x10000 && Value <= 0x10FFFF)
                {
                    UInt32 newVal = Value - 0x010000; //leaving 20 bits
                    UInt16 high = (UInt16)((newVal >> 10) + 0xD800);
                    System.Diagnostics.Debug.Assert(high <= 0xDBFF && high >= 0xD800);
                    UInt16 low = (UInt16)((newVal & 0x03FF) + 0xDC00);
                    System.Diagnostics.Debug.Assert(low <= 0xDFFF && low >= 0xDC00);
                    return new[] { high, low };
                }

                throw new UnsupportedCodepointException();
            }
        }

        //https://en.wikipedia.org/wiki/UTF-8
        public IEnumerable<byte> AsUtf8
        {
            get
            {
                //up to 7 bits
                if (Value <= 0x007F)
                {
                    yield return (byte)Value;<commit_msg>IEnumerable<UInt16> for Codepoint.AsUtf16<commit_after>﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Linq;
using System.Globalization;

namespace NeoSmart.Unicode
{
    public struct Codepoint : IComparable<Codepoint>, IComparable<UInt32>, IEquatable<Codepoint>, 
        IEquatable<string>, IComparable<string>, IEquatable<char>
    {
        public readonly UInt32 Value;

        public Codepoint(UInt32 value)
        {
            Value = value;
        }

        public Codepoint(long value)
            : this ((UInt32) value)
        { }

        /// <summary>
        /// Create a unicode codepoint from hexadecimal representation, supporting U+xxxx and 0xYYYY notation.
        /// </summary>
        /// <param name="hexValue"></param>
        public Codepoint(string hexValue)
        {
            if ((hexValue.StartsWith("0x") || hexValue.StartsWith("U+") || hexValue.StartsWith("u+")))
            {
                hexValue = hexValue.Substring(2);
            }
            if (!uint.TryParse(hexValue, NumberStyles.HexNumber, CultureInfo.CurrentCulture.NumberFormat, out Value))
            {
                throw new UnsupportedCodepointException();
            }
        }

        public UInt32 AsUtf32 => Value;

        //https://en.wikipedia.org/wiki/UTF-16
        public IEnumerable<UInt16> AsUtf16
        {
           get
            {
                //U+0000 to U+D7FF and U+E000 to U+FFFF
                if (Value <= 0xFFFF)
                {
                    yield return (UInt16)Value;
                }
                //U+10000 to U+10FFFF
                else if (Value >= 0x10000 && Value <= 0x10FFFF)
                {
                    UInt32 newVal = Value - 0x010000; //leaving 20 bits
                    UInt16 high = (UInt16) ((newVal >> 10) + 0xD800);
                    System.Diagnostics.Debug.Assert(high <= 0xDBFF && high >= 0xD800);
                    yield return high;

                    UInt16 low = (UInt16) ((newVal & 0x03FF) + 0xDC00);
                    System.Diagnostics.Debug.Assert(low <= 0xDFFF && low >= 0xDC00);
                    yield return low;
                }
                else
                {
                    throw new UnsupportedCodepointException();
                }
            }
        }

        //https://en.wikipedia.org/wiki/UTF-8
        public IEnumerable<byte> AsUtf8
        {
            get
            {
                //up to 7 bits
                if (Value <= 0x007F)
                {
                    yield return (byte)Value;
{'fraction_non_alphanumeric': 0.10529446757882213, 'fraction_numerical': 0.039064049177077134, 'mean_word_length': 1.4296724470134874, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 39, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5484230', 'n_tokens_mistral': 1543, 'n_tokens_neox': 1378, 'n_words': 302}