<commit_before>﻿//* 
//* Copyright (C) 2005-2006  <NAME>
//*
//* This library is free software; you can redistribute it and/or
//* modify it under the terms of the GNU Lesser General Public
//* License as published by the Free Software Foundation; either
//* version 2.1 of the License, or (at your option) any later version.
//*
//* This library is distributed in the hope that it will be useful,
//* but WITHOUT ANY WARRANTY; without even the implied warranty of
//* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//* Lesser General Public License for more details.
//*
//* You should have received a copy of the GNU Lesser General Public
//* License along with this library; if not, write to the Free Software
//* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//* 
//* To contact the project administrators for the bobo-browse project, 
//* please go to https://sourceforge.net/projects/bobo-browse/, or 
//* send mail to <EMAIL>. 

namespace BoboBrowse.Net.Facets.Filter
{
    using BoboBrowse.Net.DocIdSet;
    using BoboBrowse.Net.Facets.Data;
    using BoboBrowse.Net.Util;
    using Lucene.Net.Index;
    using Lucene.Net.Search;
    using System;

    public class MultiValueFacetFilter : RandomAccessFilter
    {
        private static long serialVersionUID = 1L;

        private readonly string _val;
        private readonly MultiDataCacheBuilder multiDataCacheBuilder;

        public MultiValueFacetFilter(MultiDataCacheBuilder multiDataCacheBuilder, string val)
        {
            this.multiDataCacheBuilder = multiDataCacheBuilder;
            _val = val;
        }

        private sealed class MultiValueFacetDocIdSetIterator : FacetFilter.FacetDocIdSetIterator
        {
            private readonly BigNestedIntArray _nestedArray;

            public MultiValueFacetDocIdSetIterator(MultiValueFacetDataCache dataCache, int index)
                : base(dataCache, index)
            {
                _nestedArray = dataCache._nestedArray;
            }           

            public override int NextDoc()
            {
                while (doc < maxID) // not yet reached end
                {
                    if (_nestedArray.Contains(++doc, index))
                    {
                        return doc;
                    }
                }
                return DocIdSetIterator.NO_MORE_DOCS;
            }

            public override int Advance(int target)
            {
                if (target > doc)
                {
                    doc = target - 1;
                    return NextDoc();
                }
                return NextDoc();
            }            
        }

        private class RandomRandomAccessDocIdSet : RandomAccessDocIdSet
        {
            private MultiValueFacetFilter parent;

            public RandomRandomAccessDocIdSet(MultiValueFacetFilter parent)
            {
                this.parent = parent;
            }

            public override DocIdSetIterator Iterator()
            {
                return new MultiValueFacetDocIdSetIterator(parent._dataCache, parent._index);
            }
            public override bool Get(int docId)
            {
                return parent._nestedArray.Contains(docId, parent._index);
            }
        }

        public override RandomAccessDocIdSet GetRandomAccessDocIdSet(IndexReader reader)
        {
            if (_index < 0)
            {
                return EmptyDocIdSet.GetInstance();
            }
            else
            {
                return new RandomRandomAccessDocIdSet(this);
            }
        }
    }
}<commit_msg>Upgraded MultiValueFacetFilter to 3.1.0.<commit_after>﻿//* 
//* Copyright (C) 2005-2006  <NAME>
//*
//* This library is free software; you can redistribute it and/or
//* modify it under the terms of the GNU Lesser General Public
//* License as published by the Free Software Foundation; either
//* version 2.1 of the License, or (at your option) any later version.
//*
//* This library is distributed in the hope that it will be useful,
//* but WITHOUT ANY WARRANTY; without even the implied warranty of
//* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//* Lesser General Public License for more details.
//*
//* You should have received a copy of the GNU Lesser General Public
//* License along with this library; if not, write to the Free Software
//* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//* 
//* To contact the project administrators for the bobo-browse project, 
//* please go to https://sourceforge.net/projects/bobo-browse/, or 
//* send mail to <EMAIL>. 

// Version compatibility level: 3.1.0
namespace BoboBrowse.Net.Facets.Filter
{
    using BoboBrowse.Net.DocIdSet;
    using BoboBrowse.Net.Facets.Data;
    using BoboBrowse.Net.Util;
    using Lucene.Net.Index;
    using Lucene.Net.Search;
    using System;

    public class MultiValueFacetFilter : RandomAccessFilter
    {
        private static long serialVersionUID = 1L;

        private readonly string _val;
        private readonly MultiDataCacheBuilder multiDataCacheBuilder;

        public MultiValueFacetFilter(MultiDataCacheBuilder multiDataCacheBuilder, string val)
        {
            this.multiDataCacheBuilder = multiDataCacheBuilder;
            _val = val;
        }

        public double GetFacetSelectivity(BoboIndexReader reader)
        {
            double selectivity = 0;
            MultiValueFacetDataCache dataCache = MultiDataCacheBuilder.Build(reader);
            int idx = dataCache.ValArray.IndexOf(_val);
            if (idx < 0)
            {
                return 0.0;
            }
            int freq = dataCache.freqs[idx];
            int total = reader.MaxDoc;
            selectivity = (double)freq / (double)total;
            return selectivity;
        }

        private sealed class MultiValueFacetDocIdSetIterator : FacetFilter.FacetDocIdSetIterator
        {
            private readonly BigNestedIntArray _nestedArray;

            public MultiValueFacetDocIdSetIterator(MultiValueFacetDataCache dataCache, int index)
                : base(dataCache, index)
            {
                _nestedArray = dataCache.NestedArray;
            }           

            public override int NextDoc()
            {
                _doc = (_doc < _maxID ? _nestedArray.FindValue(_index, (_doc + 1), _maxID) : NO_MORE_DOCS));
                return _doc;
            }

            public override int Advance(int id)
            {
                if (_doc < id)
                {
                    doc = (id <= _maxID ? _nestedArray.FindValue(_index, id, _maxID) : NO_MORE_DOCS));
                }
                return NextDoc();
            }            
        }

        public override RandomAccessDocIdSet GetRandomAccessDocIdSet(BoboIndexReader reader)
        {
            MultiValueFacetDataCache dataCache = multiDataCacheBuilder.Build(reader);
            int index = dataCache.ValArray.IndexOf(_val);
            BigNestedIntArray nestedArray = dataCache.NestedArray;
            if (index < 0)
            {
                return EmptyDocIdSet.GetInstance();
            }
            else
            {
                return new MultiValueRandomAccessDocIdSet(dataCache, index, nestedArray);
            }
        }

        private class MultiValueRandomAccessDocIdSet : RandomAccessDocIdSet
        {
            private readonly MultiValueFacetDataCache _dataCache;
            private readonly int _index;
            private readonly BigNestedIntArray _nestedArray;

            public MultiValueRandomAccessDocIdSet(MultiValueFacetDataCache dataCache, int index, BigNestedIntArray nestedArray)
            {
                _dataCache = dataCache;
                _index = index;
                _nestedArray = nestedArray;
            }

            public override DocIdSetIterator Iterator()
            {
                return new MultiValueFacetDocIdSetIterator(_dataCache, _index);
            }
            public override bool Get(int docId)
            {
                return _nestedArray.Contains(docId, _index);
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.07020109689213894, 'fraction_numerical': 0.007068860450944546, 'mean_word_length': 1.870234347674012, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4227202', 'n_tokens_mistral': 2168, 'n_tokens_neox': 2039, 'n_words': 606}