<commit_before>﻿using Lucene.Net.Support;
using System;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
using System.Threading;

namespace Lucene.Net.Facet.Taxonomy
{
    /*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    using ArrayUtil = Lucene.Net.Util.ArrayUtil;
    using AtomicReaderContext = Lucene.Net.Index.AtomicReaderContext;
    using BinaryDocValues = Lucene.Net.Index.BinaryDocValues;
    using DocValuesFormat = Lucene.Net.Codecs.DocValuesFormat;
    using IAccountable = Lucene.Net.Util.IAccountable;
    using Int32sRef = Lucene.Net.Util.Int32sRef;
    using RamUsageEstimator = Lucene.Net.Util.RamUsageEstimator;

    /// <summary>
    /// A per-segment cache of documents' facet ordinals. Every
    /// <see cref="CachedOrds"/> holds the ordinals in a raw <see cref="T:int[]"/>, 
    /// and therefore consumes as much RAM as the total
    /// number of ordinals found in the segment, but saves the
    /// CPU cost of decoding ordinals during facet counting.
    /// 
    /// <para>
    /// <b>NOTE:</b> every <see cref="CachedOrds"/> is limited to 2.1B
    /// total ordinals. If that is a limitation for you then
    /// consider limiting the segment size to fewer documents, or
    /// use an alternative cache which pages through the category
    /// ordinals.
    /// 
    /// </para>
    /// <para>
    /// <b>NOTE:</b> when using this cache, it is advised to use
    /// a <see cref="DocValuesFormat"/> that does not cache the data in
    /// memory, at least for the category lists fields, or
    /// otherwise you'll be doing double-caching.
    /// 
    /// </para>
    /// <para>
    /// <b>NOTE:</b> create one instance of this and re-use it
    /// for all facet implementations (the cache is per-instance,
    /// not static).
    /// </para>
    /// </summary>
    public class CachedOrdinalsReader : OrdinalsReader, IAccountable
    {
        private readonly OrdinalsReader source;

#if FEATURE_CONDITIONALWEAKTABLE_ENUMERATOR
        private readonly ConditionalWeakTable<object, CachedOrds> ordsCache = new ConditionalWeakTable<object, CachedOrds>();
        private readonly object ordsCacheLock = new object();
#else
        private readonly WeakDictionary<object, CachedOrds> ordsCache = new WeakDictionary<object, CachedOrds>();
        private readonly ReaderWriterLockSlim syncLock = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
#endif

        /// <summary>
        /// Sole constructor. </summary>
        public CachedOrdinalsReader(OrdinalsReader source)
        {
            this.source = source;
        }

        private CachedOrds GetCachedOrds(AtomicReaderContext context)
        {
            object cacheKey = context.Reader.CoreCacheKey;
#if FEATURE_CONDITIONALWEAKTABLE_ENUMERATOR
            return ordsCache.GetValue(cacheKey, (cacheKey) => new CachedOrds(source.GetReader(context), context.Reader.MaxDoc));
#else
            syncLock.EnterUpgradeableReadLock();
            try
            {
                if (!ordsCache.TryGetValue(cacheKey, out CachedOrds ords))
                {
                    ords = new CachedOrds(source.GetReader(context), context.Reader.MaxDoc);
                    syncLock.EnterWriteLock();
                    try
                    {
                        ordsCache[cacheKey] = ords;
                    }
                    finally
                    {
                        syncLock.ExitWriteLock();
                    }
                }

                return ords;
            }
            finally
            {
                syncLock.ExitUpgradeableReadLock();
            }
#endif
        }

        public override string IndexFieldName => source.IndexFieldName;
<commit_msg>PERFORMANCE: Lucene.Net.Facet.Taxonomy.CachedOrdinalsReader: Factored out call to EnterUpgradeableReadLock() so reads can happen concurrently.<commit_after>﻿using Lucene.Net.Support;
using System;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
using System.Threading;

namespace Lucene.Net.Facet.Taxonomy
{
    /*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    using ArrayUtil = Lucene.Net.Util.ArrayUtil;
    using AtomicReaderContext = Lucene.Net.Index.AtomicReaderContext;
    using BinaryDocValues = Lucene.Net.Index.BinaryDocValues;
    using DocValuesFormat = Lucene.Net.Codecs.DocValuesFormat;
    using IAccountable = Lucene.Net.Util.IAccountable;
    using Int32sRef = Lucene.Net.Util.Int32sRef;
    using RamUsageEstimator = Lucene.Net.Util.RamUsageEstimator;

    /// <summary>
    /// A per-segment cache of documents' facet ordinals. Every
    /// <see cref="CachedOrds"/> holds the ordinals in a raw <see cref="T:int[]"/>, 
    /// and therefore consumes as much RAM as the total
    /// number of ordinals found in the segment, but saves the
    /// CPU cost of decoding ordinals during facet counting.
    /// 
    /// <para>
    /// <b>NOTE:</b> every <see cref="CachedOrds"/> is limited to 2.1B
    /// total ordinals. If that is a limitation for you then
    /// consider limiting the segment size to fewer documents, or
    /// use an alternative cache which pages through the category
    /// ordinals.
    /// 
    /// </para>
    /// <para>
    /// <b>NOTE:</b> when using this cache, it is advised to use
    /// a <see cref="DocValuesFormat"/> that does not cache the data in
    /// memory, at least for the category lists fields, or
    /// otherwise you'll be doing double-caching.
    /// 
    /// </para>
    /// <para>
    /// <b>NOTE:</b> create one instance of this and re-use it
    /// for all facet implementations (the cache is per-instance,
    /// not static).
    /// </para>
    /// </summary>
    public class CachedOrdinalsReader : OrdinalsReader, IAccountable
    {
        private readonly OrdinalsReader source;

#if FEATURE_CONDITIONALWEAKTABLE_ENUMERATOR
        private readonly ConditionalWeakTable<object, CachedOrds> ordsCache = new ConditionalWeakTable<object, CachedOrds>();
        private readonly object ordsCacheLock = new object();
#else
        private readonly WeakDictionary<object, CachedOrds> ordsCache = new WeakDictionary<object, CachedOrds>();
        private readonly ReaderWriterLockSlim syncLock = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
#endif

        /// <summary>
        /// Sole constructor. </summary>
        public CachedOrdinalsReader(OrdinalsReader source)
        {
            this.source = source;
        }

        private CachedOrds GetCachedOrds(AtomicReaderContext context)
        {
            object cacheKey = context.Reader.CoreCacheKey;
#if FEATURE_CONDITIONALWEAKTABLE_ENUMERATOR
            return ordsCache.GetValue(cacheKey, (cacheKey) => new CachedOrds(source.GetReader(context), context.Reader.MaxDoc));
#else
            CachedOrds ords;
            syncLock.EnterReadLock();
            try
            {
                if (ordsCache.TryGetValue(cacheKey, out ords))
                    return ords;
            }
            finally
            {
                syncLock.ExitReadLock();
            }

            ords = new CachedOrds(source.GetReader(context), context.Reader.MaxDoc);
            syncLock.EnterWriteLock();
            try
            {
                ordsCache[cacheKey] = ords;
            }
            finally
            {
                syncLock.ExitWriteLock();
            }

            return ords;
#endif
        }

        public override string IndexFieldName => source.IndexFieldName;

{'fraction_non_alphanumeric': 0.08985570616528203, 'fraction_numerical': 0.0021862702229995625, 'mean_word_length': 2.3353991979584396, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7316315', 'n_tokens_mistral': 2481, 'n_tokens_neox': 2399, 'n_words': 776}