<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using sones.Plugins.Index.Compound;
using sones.Plugins.Index.Abstract;
using sones.Plugins.Index.Persistent;
using sones.Plugins.Index.Fulltext;
using sones.Library.VersionedPluginManager;
using sones.Plugins.Index.Helper;
using sones.Plugins.Index.LuceneIdx;
using sones.Plugins.Index.ErrorHandling;

namespace sones.Plugins.Index.LuceneIdx
{
    public class LuceneCompoundKey : ICompoundIndexKey
    {
        private long _PropertyID;
        private String _Key;

        public LuceneCompoundKey(long myPropertyID, String myKey)
        {
            _PropertyID = myPropertyID;
            _Key = myKey;
        }

        public long PropertyID
        {
            get { return _PropertyID; }
        }

        public IComparable Key
        {
            get { return _Key; }
        }
    }

    public class SonesLuceneIndex : ASonesIndex, ISonesPersistentIndex, ISonesFulltextIndex, IPluginable
    {
        #region Data

        /// <summary>
        /// The lucene index connector.
        /// </summary>
        private LuceneIndex _LuceneIndex;

        #endregion

        #region Settings

        private const int _MaxResultsFirst = 100;

        #endregion

        #region Properties

        /// <summary>
        /// Gets the id of this index within the Solr instance.
        /// </summary>
        public String IndexId
        {
            get;
            private set;
        }

        #endregion

        #region Constructor

        public SonesLuceneIndex()
        {

        }
        
        public SonesLuceneIndex(String myIndexId, String myPersistencePath = null, IList<Int64> myPropertyIDs = null)
        {
            if (myIndexId == null)
                throw new ArgumentNullException("myIndexId");

            if (myPersistencePath == null)
            {
                _LuceneIndex = new LuceneIndex(myIndexId);
            }
            else
            {
                _LuceneIndex = new LuceneIndex(myIndexId, myPersistencePath);
            }

            IndexId = myIndexId;
        }

        #endregion

        #region ISonesPersistentIndex

        public void Dispose()
        {
            
        }

        public void Shutdown()
        {
            
        }

        #endregion

        #region ISonesFulltextIndex
        
        public ISonesFulltextResult Query(string myQuery)
        {
            var result = _LuceneIndex.GetEntries(_MaxResultsFirst, myQuery);

            if (result.TotalHits > _MaxResultsFirst)
            {
                result = _LuceneIndex.GetEntries(result.TotalHits, myQuery);
            }

            var lucene_result =  new LuceneResult(result);
            result.Close();

            return lucene_result;
        }

        public long KeyCount(long myPropertyID)
        {
            return _LuceneIndex.GetKeys(entry => entry.IndexId == this.IndexId).Count();
        }

        public IEnumerable<IComparable> Keys(long myPropertyID)
        {
            var result = _LuceneIndex.GetEntriesInnerByField(_MaxResultsFirst, "*", myPropertyID.ToString(), LuceneIndex.Fields.PROPERTY_ID);
            if (result.TotalHits > _MaxResultsFirst)
            {
                result = _LuceneIndex.GetEntriesInnerByField(result.TotalHits, "*", myPropertyID.ToString(), LuceneIndex.Fields.PROPERTY_ID);
            }

            return result.Select<LuceneEntry, IComparable>((e) => (e.Text));
        }

        public IDictionary<long, Type> GetKeyTypes()
        {
            throw new NotImplementedException();
        }

        public void Add(IEnumerable<ICompoundIndexKey> myKeys, long myVertexID, Helper.IndexAddStrategy myIndexAddStrategy = IndexAddStrategy.UNIQUE)
        {
            throw new NotImplementedException();
        }

        public void AddRange(IEnumerable<KeyValuePair<IEnumerable<ICompoundIndexKey>, long>> myKeysValuePairs, Helper.IndexAddStrategy myIndexAddStrategy = IndexAddStrategy.UNIQUE)
        {
            throw new NotImplementedException();
        }

        public bool TryGetValues(IEnumerable<ICompoundIndexKey> myKeys, out IEnumerable<long> myVertexIDs)
        {
            throw new NotImplementedException();<commit_msg>[GRAPHDB-57] implemented KeyCount and Keys methods (CompoundIndex overloads) + implemented CompoundIndex overload of AddRange method<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using sones.Plugins.Index.Compound;
using sones.Plugins.Index.Abstract;
using sones.Plugins.Index.Persistent;
using sones.Plugins.Index.Fulltext;
using sones.Library.VersionedPluginManager;
using sones.Plugins.Index.Helper;
using sones.Plugins.Index.LuceneIdx;
using sones.Plugins.Index.ErrorHandling;

namespace sones.Plugins.Index.LuceneIdx
{
    public class LuceneCompoundKey : ICompoundIndexKey
    {
        private long _PropertyID;
        private String _Key;

        public LuceneCompoundKey(long myPropertyID, String myKey)
        {
            _PropertyID = myPropertyID;
            _Key = myKey;
        }

        public long PropertyID
        {
            get { return _PropertyID; }
        }

        public IComparable Key
        {
            get { return _Key; }
        }
    }

    public class SonesLuceneIndex : ASonesIndex, ISonesPersistentIndex, ISonesFulltextIndex, IPluginable
    {
        #region Data

        /// <summary>
        /// The lucene index connector.
        /// </summary>
        private LuceneIndex _LuceneIndex;

        #endregion

        #region Settings

        private const int _MaxResultsFirst = 100;

        #endregion

        #region Properties

        /// <summary>
        /// Gets the id of this index within the Solr instance.
        /// </summary>
        public String IndexId
        {
            get;
            private set;
        }

        #endregion

        #region Constructor

        public SonesLuceneIndex()
        {

        }
        
        public SonesLuceneIndex(String myIndexId, String myPersistencePath = null, IList<Int64> myPropertyIDs = null)
        {
            if (myIndexId == null)
                throw new ArgumentNullException("myIndexId");

            if (myPersistencePath == null)
            {
                _LuceneIndex = new LuceneIndex(myIndexId);
            }
            else
            {
                _LuceneIndex = new LuceneIndex(myIndexId, myPersistencePath);
            }

            IndexId = myIndexId;
        }

        #endregion

        #region ISonesPersistentIndex

        public void Dispose()
        {
            
        }

        public void Shutdown()
        {
            
        }

        #endregion

        #region ISonesFulltextIndex
        
        public ISonesFulltextResult Query(string myQuery)
        {
            var result = _LuceneIndex.GetEntries(_MaxResultsFirst, myQuery);

            if (result.TotalHits > _MaxResultsFirst)
            {
                result = _LuceneIndex.GetEntries(result.TotalHits, myQuery);
            }

            var lucene_result =  new LuceneResult(result);
            result.Close();

            return lucene_result;
        }

        public long KeyCount(long myPropertyID)
        {
            var keys = _LuceneIndex.GetKeys(entry => (entry.IndexId == this.IndexId) && (entry.PropertyId == myPropertyID));
            var groupedkeys = keys.GroupBy(s => s);
            var count = groupedkeys.Count();
            keys.Close();

            return count;
        }

        public IEnumerable<IComparable> Keys(long myPropertyID)
        {
            var result = _LuceneIndex.GetEntriesInnerByField(_MaxResultsFirst, "*:*", myPropertyID.ToString(), LuceneIndex.Fields.PROPERTY_ID);
            if (result.TotalHits > _MaxResultsFirst)
            {
                result.Close();
                result = _LuceneIndex.GetEntriesInnerByField(result.TotalHits, "*:*", myPropertyID.ToString(), LuceneIndex.Fields.PROPERTY_ID);
            }

            // unforunately we have to breakup Lazy as interface does not support closing
            List<IComparable> ret = new List<IComparable>();

            foreach (var entry in result.Select<LuceneEntry, IComparable>((e) => (e.Text)))
            {
                ret.Add(entry);
            }

            result.Close();

            return ret;
        }

        public IDictionary<long, Type> GetKeyTypes()
        {
            throw new NotImplementedException();
        }

        public void Add(IEnumerable<ICompoundIndexKey> myKeys, long myVertexID, Helper.IndexAddStrategy myIndexAddStrategy = IndexAddStrategy.UNIQUE)
        {
            throw new NotImplementedException();
        }

        public void AddRange(IEnumerable<KeyValuePair<IEnumerable<ICompoundIndexKey>, long>> myKeysValuePairs, Helper.IndexAddStrategy myIndexAddStrategy = IndexAddStrategy.UNIQUE)
        {
            foreach (var kvp in myKeysValuePairs)
            {
                foreach (var key in kvp.Key)
                {
                    AddEntry(key.Key, new HashSet<long>() { kvp.Value }, myIndexAddStrategy, key.PropertyID);
                }
            }
        }

        public bool TryGetValues(IEnumerable<ICompoundIndexKey> myKeys, out IEnumerable<long> myVertexIDs)
        {
            throw new NotImplementedException();
{'fraction_non_alphanumeric': 0.07952138924184668, 'fraction_numerical': 0.0012706480304955528, 'mean_word_length': 1.8919167176974894, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '828862', 'n_tokens_mistral': 2579, 'n_tokens_neox': 2490, 'n_words': 538}