<commit_before>﻿using Microsoft.Extensions.Logging;
using Sir.Core;
using Sir.Documents;
using Sir.VectorSpace;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

namespace Sir.Search
{
    /// <summary>
    /// Dispatcher of sessions.
    /// </summary>
    public class SessionFactory : IDisposable, ISessionFactory
    {
        private ConcurrentDictionary<ulong, ConcurrentDictionary<ulong, long>> _keys;

        public string Directory { get; }
        public ILogger Logger { get; }

        public SessionFactory(string directory = null, ILogger logger = null)
        {
            var time = Stopwatch.StartNew();

            Directory = directory;

            if (Directory != null && !System.IO.Directory.Exists(Directory))
            {
                System.IO.Directory.CreateDirectory(Directory);
            }

            _keys = LoadKeys();
            Logger = logger;

           Log($"sessionfactory initiated in {time.Elapsed}");
        }

        private void Log(string message)
        {
            if (Logger != null)
                Logger.LogInformation(message);
        }

        public long GetDocCount(string collection)
        {
            var fileName = Path.Combine(Directory, $"{collection.ToHash()}.dix");

            if (!File.Exists(fileName))
                return 0;

            return new FileInfo(fileName).Length / (sizeof(long) + sizeof(int));
        }

        public void Truncate(ulong collectionId)
        {
            var count = 0;

            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*"))
            {
                File.Delete(file);
                count++;
            }

            _keys.Remove(collectionId, out _);

            Log($"truncated collection {collectionId} ({count} files)");
        }

        public void TruncateIndex(ulong collectionId)
        {
            var count = 0;

            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.ix"))
            {
                File.Delete(file);
                count++;
            }
            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.ixp"))
            {
                File.Delete(file);
                count++;
            }
            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.ixtp"))
            {
                File.Delete(file);
                count++;
            }
            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.vec"))
            {
                File.Delete(file);
                count++;
            }
            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.pos"))
            {
                File.Delete(file);
                count++;
            }

            Log($"truncated index {collectionId} ({count} files)");
        }

        public void Optimize(
            string collection,
            HashSet<string> selectFields, 
            ITextModel model,
            int skip = 0,
            int take = 0,
            int reportFrequency = 1000,
            int pageSize = 100000,
            bool truncateIndex = true)
        {
            var collectionId = collection.ToHash();
            var debugger = new IndexDebugger(Logger, reportFrequency);

            if (truncateIndex)
                TruncateIndex(collectionId);

            using (var docStream = new DocumentStreamSession(this))
            using (var documentReader = new DocumentReader(collectionId, this))
            {
                var payload = docStream.ReadDocs(
                        selectFields,
                        documentReader,
                        skip,
                        take);

                using (var writeQueue = new ProducerConsumerQueue<IndexSession<string>>(indexSession =>
                {
                    using (var stream = new WritableIndexStream(collectionId, this, logger: Logger))
                    {
                        stream.Write(indexSession.GetInMemoryIndex());
                    }
                }))
                {
                    foreach (var page in payload.Batch(pageSize))
                    {
                        using (var indexSession = new IndexSession<string>(model, model))
                        {
                            using (var indexQueue = new IndexProducerConsumerQueue(field =>
                            {
                                indexSession.Put(field.DocumentId, field.KeyId, field.Tokens);
                            }))
                            {
                                using (var analyzeQueue = new ProducerConsumerQueue<Field>(field =>
                                {
                                    field.Analyze(model);
                                    indexQueue.Enqueue(field);
                                }))
                                {
                                    foreach (var document in page)<commit_msg>debugging<commit_after>﻿using Microsoft.Extensions.Logging;
using Sir.Core;
using Sir.Documents;
using Sir.VectorSpace;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

namespace Sir.Search
{
    /// <summary>
    /// Dispatcher of sessions.
    /// </summary>
    public class SessionFactory : IDisposable, ISessionFactory
    {
        private ConcurrentDictionary<ulong, ConcurrentDictionary<ulong, long>> _keys;

        public string Directory { get; }
        public ILogger Logger { get; }

        public SessionFactory(string directory = null, ILogger logger = null)
        {
            var time = Stopwatch.StartNew();

            Directory = directory;

            if (Directory != null && !System.IO.Directory.Exists(Directory))
            {
                System.IO.Directory.CreateDirectory(Directory);
            }

            _keys = LoadKeys();
            Logger = logger;

           Log($"sessionfactory initiated in {time.Elapsed}");
        }

        private void Log(string message)
        {
            if (Logger != null)
                Logger.LogInformation(message);
        }

        public long GetDocCount(string collection)
        {
            var fileName = Path.Combine(Directory, $"{collection.ToHash()}.dix");

            if (!File.Exists(fileName))
                return 0;

            return new FileInfo(fileName).Length / (sizeof(long) + sizeof(int));
        }

        public void Truncate(ulong collectionId)
        {
            var count = 0;

            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*"))
            {
                File.Delete(file);
                count++;
            }

            _keys.Remove(collectionId, out _);

            Log($"truncated collection {collectionId} ({count} files)");
        }

        public void TruncateIndex(ulong collectionId)
        {
            var count = 0;

            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.ix"))
            {
                File.Delete(file);
                count++;
            }
            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.ixp"))
            {
                File.Delete(file);
                count++;
            }
            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.ixtp"))
            {
                File.Delete(file);
                count++;
            }
            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.vec"))
            {
                File.Delete(file);
                count++;
            }
            foreach (var file in System.IO.Directory.GetFiles(Directory, $"{collectionId}*.pos"))
            {
                File.Delete(file);
                count++;
            }

            Log($"truncated index {collectionId} ({count} files)");
        }

        public void Optimize(
            string collection,
            HashSet<string> selectFields, 
            ITextModel model,
            int skip = 0,
            int take = 0,
            int reportFrequency = 1000,
            int pageSize = 100000,
            bool truncateIndex = true)
        {
            var collectionId = collection.ToHash();
            var debugger = new IndexDebugger(Logger, reportFrequency);

            if (truncateIndex)
                TruncateIndex(collectionId);

            using (var docStream = new DocumentStreamSession(this))
            using (var documentReader = new DocumentReader(collectionId, this))
            {
                var payload = docStream.ReadDocs(
                        selectFields,
                        documentReader,
                        skip,
                        take);

                using (var writeQueue = new ProducerConsumerQueue<IndexSession<string>>(indexSession =>
                {
                    using (var stream = new WritableIndexStream(collectionId, this, logger: Logger))
                    {
                        stream.Write(indexSession.GetInMemoryIndex());

                        debugger.Step(indexSession, pageSize);
                    }
                }))
                {
                    foreach (var page in payload.Batch(pageSize))
                    {
                        using (var indexSession = new IndexSession<string>(model, model))
                        {
                            using (var indexQueue = new IndexProducerConsumerQueue(field =>
                            {
                                indexSession.Put(field.DocumentId, field.KeyId, field.Tokens);
                            }))
                            {
                                using (var analyzeQueue = new ProducerConsumerQueue<Field>(field =>
                                {
                                    field.Analyze(model);
                                    indexQueue.Enqueue(field);
                                }))
                                {
                                    foreach (var document in page)
{'fraction_non_alphanumeric': 0.09357568763223696, 'fraction_numerical': 0.0028851702250432777, 'mean_word_length': 1.341589732042333, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4029127', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2365, 'n_words': 539}