<commit_before>using Lucene.Net.Support;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;

namespace Lucene.Net.Util
{
    /*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    /// <summary>
    /// Represents byte[], as a slice (offset + length) into an
    ///  existing byte[].  The <seealso cref="#bytes"/> member should never be null;
    ///  use <seealso cref="#EMPTY_BYTES"/> if necessary.
    ///
    /// <p><b>Important note:</b> Unless otherwise noted, Lucene uses this class to
    /// represent terms that are encoded as <b>UTF8</b> bytes in the index. To
    /// convert them to a Java <seealso cref="String"/> (which is UTF16), use <seealso cref="#utf8ToString"/>.
    /// Using code like {@code new String(bytes, offset, length)} to do this
    /// is <b>wrong</b>, as it does not respect the correct character set
    /// and may return wrong results (depending on the platform's defaults)!
    /// </summary>
    public sealed class BytesRef : IComparable // LUCENENET TODO: Implement IComparable<BytesRef>
    {
        /// <summary>
        /// An empty byte array for convenience </summary>
        public static readonly byte[] EMPTY_BYTES = new byte[0];

        /// <summary>
        /// The contents of the BytesRef. Should never be {@code null}.
        /// </summary>
        [WritableArray]
        public byte[] Bytes
        {
            get { return bytes; }
            set
            {
                if (value == null)
                {
                    throw new ArgumentNullException("bytes may never be set to null");
                }
                bytes = value;
            }
        }
        private byte[] bytes;

        /// <summary>
        /// Offset of first valid byte.
        /// </summary>
        public int Offset { get; set; }

        /// <summary>
        /// Length of used bytes.
        /// </summary>
        public int Length { get; set; }

        /// <summary>
        /// Create a BytesRef with <seealso cref="#EMPTY_BYTES"/> </summary>
        public BytesRef()
            : this(EMPTY_BYTES)
        {
        }
<commit_msg>Lucene.Net.Core.Util.BytesRef: Removed validation on setting Bytes property to non-null because some tests actually set it to null<commit_after>using Lucene.Net.Support;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;

namespace Lucene.Net.Util
{
    /*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    /// <summary>
    /// Represents byte[], as a slice (offset + length) into an
    ///  existing byte[].  The <seealso cref="#bytes"/> member should never be null;
    ///  use <seealso cref="#EMPTY_BYTES"/> if necessary.
    ///
    /// <p><b>Important note:</b> Unless otherwise noted, Lucene uses this class to
    /// represent terms that are encoded as <b>UTF8</b> bytes in the index. To
    /// convert them to a Java <seealso cref="String"/> (which is UTF16), use <seealso cref="#utf8ToString"/>.
    /// Using code like {@code new String(bytes, offset, length)} to do this
    /// is <b>wrong</b>, as it does not respect the correct character set
    /// and may return wrong results (depending on the platform's defaults)!
    /// </summary>
    public sealed class BytesRef : IComparable // LUCENENET TODO: Implement IComparable<BytesRef>
    {
        /// <summary>
        /// An empty byte array for convenience </summary>
        public static readonly byte[] EMPTY_BYTES = new byte[0];

        /// <summary>
        /// The contents of the BytesRef. Should never be {@code null}.
        /// </summary>
        [WritableArray]
        public byte[] Bytes
        {
            get { return bytes; }
            set { bytes = value; } // LUCENENET NOTE: Although the comments state this cannot be null, some of the tests depend on setting it to null!
        }
        private byte[] bytes;

        /// <summary>
        /// Offset of first valid byte.
        /// </summary>
        public int Offset { get; set; }

        /// <summary>
        /// Length of used bytes.
        /// </summary>
        public int Length { get; set; }

        /// <summary>
        /// Create a BytesRef with <seealso cref="#EMPTY_BYTES"/> </summary>
        public BytesRef()
            : this(EMPTY_BYTES)
        {
        }

{'fraction_non_alphanumeric': 0.10455538746007732, 'fraction_numerical': 0.0030257186081694403, 'mean_word_length': 2.1448202959830867, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4994926', 'n_tokens_mistral': 1555, 'n_tokens_neox': 1535, 'n_words': 649}