<commit_before>﻿//*
//* Bobo Browse Engine - High performance faceted/parametric search implementation 
//* that handles various types of semi-structured data.  Written in Java.
//* 
//* Copyright (C) 2005-2006  <NAME>
//*
//* This library is free software; you can redistribute it and/or
//* modify it under the terms of the GNU Lesser General Public
//* License as published by the Free Software Foundation; either
//* version 2.1 of the License, or (at your option) any later version.
//*
//* This library is distributed in the hope that it will be useful,
//* but WITHOUT ANY WARRANTY; without even the implied warranty of
//* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//* Lesser General Public License for more details.
//*
//* You should have received a copy of the GNU Lesser General Public
//* License along with this library; if not, write to the Free Software
//* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//* 
//* To contact the project administrators for the bobo-browse project, 
//* please go to https://sourceforge.net/projects/bobo-browse/, or 
//* send mail to <EMAIL>.  

namespace LuceneExt.Impl
{
    using System;
    using System.IO;
    using Lucene.Net.Search;
    using LuceneExt.Util;

    [Serializable]
    public class IntArrayDocIdSet : DocSet
    {      

        private readonly IntArray array;

        private int pos = -1;

        public IntArrayDocIdSet(int length)
        {
            array = new IntArray(length);
        }

        public IntArrayDocIdSet()
        {
            array = new IntArray();
        }

        public override void AddDoc(int docid)
        {
            ++pos;
            array.Add(docid);
        }

        public override bool IsCacheable
        {
            get
            {
                return true;
            }
        }

        protected internal virtual int BinarySearchForNearest(int val, int begin, int end)
        {
            int mid = (begin + end) / 2;
            int midval = array.Get(mid);

            if (mid == end)
            {
                return midval >= val ? mid : -1;
            }

            if (midval < val)
            {
                // Find number equal or greater than the target.
                if (array.Get(mid + 1) >= val)
                {
                    return mid + 1;
                }

                return BinarySearchForNearest(val, mid + 1, end);
            }
            else
            {
                // Find number equal or greater than the target.
                if (midval == val)
                {
                    return mid;
                }

                return BinarySearchForNearest(val, begin, mid);
            }
        }

        internal class IntArrayDocIdSetIterator : StatefulDSIterator
        {
            private int lastReturn = -1;
            private int cursor = -1;

            private readonly IntArrayDocIdSet parent;

            public IntArrayDocIdSetIterator(IntArrayDocIdSet parent)
            {
                this.parent = parent;
                if (parent.pos == -1)
                {
                    lastReturn = DocIdSetIterator.NO_MORE_DOCS;
                }
            }

            public override int DocID()
            {
                return lastReturn;
            }

            public override int NextDoc()
            {
                if (cursor < parent.pos)
                {
                    return (lastReturn = parent.array.Get(++cursor));
                }
                return (lastReturn = DocIdSetIterator.NO_MORE_DOCS);
            }

            public override int Advance(int target)
            {
                if (lastReturn == DocIdSetIterator.NO_MORE_DOCS)
                {
                    return DocIdSetIterator.NO_MORE_DOCS;
                }

                if (target <= lastReturn)
                {
                    target = lastReturn + 1;
                }

                int end = Math.Min(cursor + (target - lastReturn), parent.pos);
                int index = parent.BinarySearchForNearest(target, cursor + 1, end);

                if (index == -1)
                {
                    cursor = parent.pos;
                    return (lastReturn = DocIdSetIterator.NO_MORE_DOCS);
                }
                else
                {
                    cursor = index;
                    return (lastReturn = parent.array.Get(cursor));
                }
            }

            public override int GetCursor()
            {
                return cursor;
            }
        }

        public override DocIdSetIterator Iterator()
        {
            return new IntArrayDocIdSetIterator(this);
        }

        public override int Size()
        {
            return pos + 1;
        }

        public override int FindWithIndex(int val)
        {
            IntArrayDocIdSetIterator dcit = new IntArrayDocIdSetIterator(this);
            try
            {
                int docid = dcit.Advance(val);
                if (docid == val)
                {
                    return dcit.GetCursor();
                }
            }
            catch (IOException e)
            {
                //
            }
            return -1;
        }
<commit_msg>Upgraded the LuceneExt.Net.Impl.IntArrayDocIdSet to Kamikaze version 3.0.6 compatibility.<commit_after>﻿﻿// Kamikaze version compatibility level: 3.0.6
namespace LuceneExt.Impl
{
    using System;
    using System.IO;
    using Lucene.Net.Search;
    using LuceneExt.Util;

    [Serializable]
    public class IntArrayDocIdSet : DocSet
    {      

        private readonly IntArray array;

        private int pos = -1;

        public IntArrayDocIdSet(int length)
        {
            array = new IntArray(length);
        }

        public IntArrayDocIdSet()
        {
            array = new IntArray();
        }

        public override void AddDoc(int docid)
        {
            ++pos;
            array.Add(docid);
        }

        public override bool IsCacheable
        {
            get
            {
                return true;
            }
        }

        protected internal virtual int BinarySearchForNearest(int val, int begin, int end)
        {
            int mid = (begin + end) / 2;
            int midval = array.Get(mid);

            if (mid == end)
            {
                return midval >= val ? mid : -1;
            }

            if (midval < val)
            {
                // Find number equal or greater than the target.
                if (array.Get(mid + 1) >= val) return mid + 1;

                return BinarySearchForNearest(val, mid + 1, end);
            }
            else
            {
                // Find number equal or greater than the target.
                if (midval == val) return mid;

                return BinarySearchForNearest(val, begin, mid);
            }
        }

        internal class IntArrayDocIdSetIterator : StatefulDSIterator
        {
            private int lastReturn = -1;

            private int cursor = -1;

            private readonly IntArrayDocIdSet parent;

            public IntArrayDocIdSetIterator(IntArrayDocIdSet parent)
            {
                this.parent = parent;
                if (parent.pos == -1) lastReturn = DocIdSetIterator.NO_MORE_DOCS;
            }

            public override int DocID()
            {
                return lastReturn;
            }

            public override int NextDoc()
            {
                if (cursor < parent.pos)
                {
                    return (lastReturn = parent.array.Get(++cursor));
                }
                return (lastReturn = DocIdSetIterator.NO_MORE_DOCS);
            }

            public override int Advance(int target)
            {
                if (lastReturn == DocIdSetIterator.NO_MORE_DOCS) return DocIdSetIterator.NO_MORE_DOCS;

                if (target <= lastReturn) target = lastReturn + 1;

                int end = Math.Min(cursor + (target - lastReturn), parent.pos);
                int index = parent.BinarySearchForNearest(target, cursor + 1, end);

                if (index == -1)
                {
                    cursor = parent.pos;
                    return (lastReturn = DocIdSetIterator.NO_MORE_DOCS);
                }
                else
                {
                    cursor = index;
                    return (lastReturn = parent.array.Get(cursor));
                }
            }

            public override int GetCursor()
            {
                return cursor;
            }
        }

        public override DocIdSetIterator Iterator()
        {
            return new IntArrayDocIdSetIterator(this);
        }

        public override int Size()
        {
            return pos + 1;
        }

        public override int FindWithIndex(int val)
        {
            IntArrayDocIdSetIterator dcit = new IntArrayDocIdSetIterator(this);
            try
            {
                int docid = dcit.Advance(val);
                if (docid == val)
                    return dcit.GetCursor();
            }
            catch (IOException e)
            {
            }
            return -1;
        }

{'fraction_non_alphanumeric': 0.07741239892183288, 'fraction_numerical': 0.005929919137466307, 'mean_word_length': 1.2122585261149534, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5889200', 'n_tokens_mistral': 2447, 'n_tokens_neox': 2251, 'n_words': 660}