<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;

namespace Sir.Store
{
    /// <summary>
    /// Read (reduce) postings.
    /// </summary>
    public class PostingsReader : ILogger
    {
        private readonly Stream _stream;

        public PostingsReader(Stream stream)
        {
            _stream = stream;
        }

        public ScoredResult Reduce(IList<Query> query, int skip, int take)
        {
            var timer = Stopwatch.StartNew();

            IDictionary<long, float> result = null;

            foreach (var q in query)
            {
                var cursor = q;

                while (cursor != null)
                {
                    var docIds = Read(cursor.PostingsOffsets, cursor.Score);

                    if (cursor.And)
                    {
                        if (result == null)
                        {
                            result = docIds;
                        }
                        else
                        {
                            var section = new Dictionary<long, float>();

                            foreach (var doc in result)
                            {
                                float score;
                                if (docIds.TryGetValue(doc.Key, out score))
                                {
                                    section.Add(doc.Key, doc.Value + score);
                                }
                            }

                            result = section;
                        }
                    }
                    else if (cursor.Not)
                    {
                        if (result != null)
                        {
                            foreach (var id in docIds.Keys)
                            {
                                result.Remove(id);
                            }
                        }
                    }
                    else // Or
                    {
                        if (result == null)
                        {
                            result = docIds;
                        }
                        else
                        {
                            foreach (var doc in docIds)
                            {
                                if (result.ContainsKey(doc.Key))
                                {
                                    result[doc.Key] += doc.Value;
                                }
                            }
                        }
                    }

                    cursor = cursor.NextTermInClause;
                }
            }

            var sortedByScore = new List<KeyValuePair<long, float>>(result);
            sortedByScore.Sort(
                delegate (KeyValuePair<long, float> pair1,
                KeyValuePair<long, float> pair2)
                {
                    return pair2.Value.CompareTo(pair1.Value);
                }
            );

            var index = skip > 0 ? skip : 0;
            var count = Math.Min(sortedByScore.Count, (take > 0 ? take : sortedByScore.Count));

            this.Log("reducing {0} into {1} docs took {2}", query, sortedByScore.Count, timer.Elapsed);

            return new ScoredResult { SortedDocuments = sortedByScore.GetRange(index, count), Total = sortedByScore.Count };
        }

        private IDictionary<long, float> Read(IList<long> offsets, float score)
        {
            var result = new Dictionary<long, float>();

            foreach(var offset in offsets)
            {
                GetPostingsFromStream(offset, result, score);
            }

            return result;
        }

        private void GetPostingsFromStream(long postingsOffset, IDictionary<long, float> result, float score)
        {
            _stream.Seek(postingsOffset, SeekOrigin.Begin);

            Span<byte> buf = stackalloc byte[sizeof(long)];

            _stream.Read(buf);

            var numOfPostings = BitConverter.ToInt64(buf);

            Span<byte> listBuf = stackalloc byte[sizeof(long) * (int)numOfPostings];

            _stream.Read(listBuf);

            foreach (var word in MemoryMarshal.Cast<byte, long>(listBuf))
            {
                result.Add(word, score);
            }
        }
    }

    public class ScoredResult
    {
        public IList<KeyValuePair<long, float>> SortedDocuments { get; set; }
        public int Total { get; set; }
    }
}<commit_msg>try fix logical errors in boolean engine<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;

namespace Sir.Store
{
    /// <summary>
    /// Read (reduce) postings.
    /// </summary>
    public class PostingsReader : ILogger
    {
        private readonly Stream _stream;

        public PostingsReader(Stream stream)
        {
            _stream = stream;
        }

        public ScoredResult Reduce(IList<Query> query, int skip, int take)
        {
            var timer = Stopwatch.StartNew();

            IDictionary<long, float> result = null;

            foreach (var q in query)
            {
                var cursor = q;

                while (cursor != null)
                {
                    var subResult = Read(cursor.PostingsOffsets, cursor.Score);

                    this.Log($"term {cursor.Term} exists in {subResult.Count} documents");

                    if (cursor.And)
                    {
                        if (result == null)
                        {
                            result = subResult;
                        }
                        else
                        {
                            var product = new Dictionary<long, float>();

                            foreach (var doc in result)
                            {
                                float score;
                                if (subResult.TryGetValue(doc.Key, out score))
                                {
                                    product.Add(doc.Key, doc.Value + score);
                                }
                            }

                            foreach (var doc in subResult)
                            {
                                float score;
                                if (result.TryGetValue(doc.Key, out score))
                                {
                                    if (!product.ContainsKey(doc.Key))
                                        product.Add(doc.Key, doc.Value + score);
                                }
                            }

                            result = product;
                        }
                    }
                    else if (cursor.Not)
                    {
                        if (result != null)
                        {
                            foreach (var id in subResult.Keys)
                            {
                                result.Remove(id);
                            }
                        }
                    }
                    else // Or
                    {
                        if (result == null)
                        {
                            result = subResult;
                        }
                        else
                        {
                            var product = new Dictionary<long, float>();

                            foreach (var doc in result)
                            {
                                float score;
                                if (subResult.TryGetValue(doc.Key, out score))
                                {
                                    product.Add(doc.Key, doc.Value + score);
                                }
                                else
                                {
                                    product.Add(doc.Key, doc.Value);
                                }
                            }

                            foreach (var doc in subResult)
                            {
                                float score;
                                if (result.TryGetValue(doc.Key, out score))
                                {
                                    if (!product.ContainsKey(doc.Key))
                                        product.Add(doc.Key, doc.Value + score);
                                }
                                else
                                {
                                    product.Add(doc.Key, doc.Value);
                                }
                            }

                            result = product;
                        }
                    }

                    cursor = cursor.NextTermInClause;
                }
            }

            var sortedByScore = new List<KeyValuePair<long, float>>(result);
            sortedByScore.Sort(
                delegate (KeyValuePair<long, float> pair1,
                KeyValuePair<long, float> pair2)
                {
                    return pair2.Value.CompareTo(pair1.Value);
                }
            );

            var index = skip > 0 ? skip : 0;
            var count = Math.Min(sortedByScore.Count, take > 0 ? take : sortedByScore.Count);

            this.Log($"found total of {sortedByScore.Count} docs in {timer.Elapsed}. skip {skip} take {take} index {index} count {count}");

            return new ScoredResult { SortedDocuments = sortedByScore.GetRange(index, count), Total = sortedByScore.Count };
        }

        private IDictionary<long, float> Read(IList<long> offsets, float score)
        {
            var result = new Dictionary<long, float>();

            foreach (var offset in offsets)
            {
                GetPostingsFromStream(offset, result, score);
            }

            return result;
        }

        private void GetPostingsFromStream(long postingsOffset, IDictionary<long, float> result, float score)
        {
            _stream.Seek(postingsOffset, SeekOrigin.Begin);

            Span<byte> buf = stackalloc byte[sizeof(long)];

            _stream.Read(buf);

            var numOfPostings = BitConverter.ToInt64(buf);

            Span<byte> listBuf = stackalloc byte[sizeof(long) * (int)numOfPostings];

            var read = _stream.Read(listBuf);

            if (read < listBuf.Length)
            {
                throw new Exception("oh dear");
            }

            foreach (var word in MemoryMarshal.Cast<byte, long>(listBuf))
            {
                result.Add(word, score);
            }
        }
    }

    public class ScoredResult
    {
        public IList<KeyValuePair<long, float>> SortedDocuments { get; set; }
        public int Total { get; set; }
    }
}
{'fraction_non_alphanumeric': 0.07837310799517133, 'fraction_numerical': 0.0019500417866097131, 'mean_word_length': 0.8537005163511188, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5376575', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2323, 'n_words': 545}