<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using log4net;
using Resin.IO.Read;
using Resin.Querying;
using Resin.IO;
using DocumentTable;

namespace Resin
{
    public class Collector : IDisposable
    {
        private static readonly ILog Log = LogManager.GetLogger(typeof(Collector));
        private readonly string _directory;
        private readonly IScoringSchemeFactory _scorerFactory;
        private readonly IDictionary<Query, IList<DocumentScore>> _scoreCache;
        private readonly IReadSession _readSession;

        public Collector(string directory, IReadSession readSession, IScoringSchemeFactory scorerFactory = null)
        {
            _readSession = readSession;
            _directory = directory;
            _scorerFactory = scorerFactory;
            _scoreCache = new Dictionary<Query, IList<DocumentScore>>();
        }

        public DocumentScore[] Collect(IList<QueryContext> query)
        {
            var scoreTime = new Stopwatch();
            scoreTime.Start();

            foreach (var clause in query)
            {
                Scan(clause);
            }

            Log.DebugFormat("scored query {0} in {1}", query, scoreTime.Elapsed);

            var reduceTime = new Stopwatch();
            reduceTime.Start();

            var reduced = query.Reduce().ToArray();

            Log.DebugFormat("reduced query {0} producing {1} scores in {2}", query, reduced.Length, scoreTime.Elapsed);

            return reduced;
        }

        private void Scan(QueryContext ctx)
        {
            var time = new Stopwatch();
            time.Start();

            if (ctx.Query is TermQuery)
            {
                TermScan(ctx);
            }
            else if (ctx.Query is PhraseQuery)
            {
                PhraseScan(ctx);
            }
            else
            {
                RangeScan(ctx);
            }

            if (Log.IsDebugEnabled && ctx.Terms.Count > 1)
            {
                Log.DebugFormat("expanded {0}: {1}",
                    ctx.Query.Value, string.Join(" ", ctx.Terms.Select(t => t.Word.Value)));
            }

            Log.DebugFormat("scanned {0} in {1}", ctx.Query.Serialize(), time.Elapsed);
        }

        private void TermScan(QueryContext ctx)
        {
            IList<Term> terms;

            using (var reader = GetTreeReader(ctx.Query.Field))
            {
                if (ctx.Query.Fuzzy)
                {
                    terms = reader.SemanticallyNear(ctx.Query.Value, ctx.Query.Edits(ctx.Query.Value))
                        .ToTerms(ctx.Query.Field);
                }
                else if (ctx.Query.Prefix)
                {
                    terms = reader.StartsWith(ctx.Query.Value)
                        .ToTerms(ctx.Query.Field);
                }
                else
                {
                    terms = reader.IsWord(ctx.Query.Value)
                        .ToTerms(ctx.Query.Field);
                }
            }
            ctx.Terms = terms;
            ctx.Postings = GetPostings(ctx);
            ctx.Scored = Score(ctx.Postings);
        }

        private void PhraseScan(QueryContext query)
        {
            var tokens = ((PhraseQuery)query.Query).Values;

            var termMatrix = new IList<Term>[tokens.Count];

            for (int index = 0;index < tokens.Count; index++)
            {
                var token = tokens[index];
                IList<Term> terms;

                using (var reader = GetTreeReader(query.Query.Field))
                {
                    if (query.Query.Fuzzy)
                    {
                        terms = reader.SemanticallyNear(token, query.Query.Edits(token))
                            .ToTerms(query.Query.Field);
                    }
                    else if (query.Query.Prefix)
                    {
                        terms = reader.StartsWith(token)
                            .ToTerms(query.Query.Field);
                    }
                    else
                    {
                        terms = reader.IsWord(token)
                            .ToTerms(query.Query.Field);
                    }
                }

                termMatrix[index] = terms;
            }
        }

        private void RangeScan(QueryContext ctx)
        {
            using (var reader = GetTreeReader(ctx.Query.Field))
            {
                ctx.Terms = reader.Range(ctx.Query.Value, ((RangeQuery)ctx.Query).ValueUpperBound)
                        .ToTerms(ctx.Query.Field);
            }

            ctx.Postings = GetPostings(ctx);
            ctx.Scored = Score(ctx.Postings);
        }

        private IList<DocumentPosting> GetPostings(QueryContext query)
        {
            var time = Stopwatch.StartNew();

            var postings = GetPostings(query.Terms);

            Log.DebugFormat("read postings for {0} in {1}", query.Query.Serialize(), time.Elapsed);

            return postings;
        }

        private IList<DocumentPosting> GetPostings(IList<Term> terms)
        {
            var postings = terms.Count > 0 ? _readSession.ReadPostings(terms) : null;

            IList<DocumentPosting> reduced;

            if (postings == null)
            {
                reduced = new DocumentPosting[0];
            }
            else
            {
                reduced = postings.Sum();
            }

            return reduced;
        }

        private IList<DocumentScore> Score(IList<DocumentPosting> postings)
        {
            var scores = new List<DocumentScore>(postings.Count);

            if (postings != null)
            {
                if (_scorerFactory == null)
                {
                    foreach (var posting in postings)
                    {
                        var docHash = _readSession.ReadDocHash(posting.DocumentId);

                        if (!docHash.IsObsolete)
                        {
                            scores.Add(new DocumentScore(posting.DocumentId, docHash.Hash, 0, _readSession.Version));
                        }
                    }
                }
                else
                {
                    if (postings.Any())
                    {
                        var docsWithTerm = postings.Count;<commit_msg>create weighted word vectors at query time (phrase query)<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using log4net;
using Resin.IO.Read;
using Resin.Querying;
using Resin.IO;
using DocumentTable;

namespace Resin
{
    public class Collector : IDisposable
    {
        private static readonly ILog Log = LogManager.GetLogger(typeof(Collector));
        private readonly string _directory;
        private readonly IScoringSchemeFactory _scorerFactory;
        private readonly IDictionary<Query, IList<DocumentScore>> _scoreCache;
        private readonly IReadSession _readSession;

        public Collector(string directory, IReadSession readSession, IScoringSchemeFactory scorerFactory = null)
        {
            _readSession = readSession;
            _directory = directory;
            _scorerFactory = scorerFactory;
            _scoreCache = new Dictionary<Query, IList<DocumentScore>>();
        }

        public DocumentScore[] Collect(IList<QueryContext> query)
        {
            var scoreTime = Stopwatch.StartNew();

            foreach (var clause in query)
            {
                Scan(clause);
                Score(clause);
            }

            Log.DebugFormat("scored query {0} in {1}", query, scoreTime.Elapsed);

            var reduceTime = new Stopwatch();
            reduceTime.Start();

            var reduced = query.Reduce().ToArray();

            Log.DebugFormat("reduced query {0} producing {1} scores in {2}", query, reduced.Length, scoreTime.Elapsed);

            return reduced;
        }

        private void Scan(QueryContext ctx)
        {
            var time = new Stopwatch();
            time.Start();

            if (ctx.Query is TermQuery)
            {
                TermScan(ctx);
            }
            else if (ctx.Query is PhraseQuery)
            {
                PhraseScan(ctx);
            }
            else
            {
                RangeScan(ctx);
            }

            if (Log.IsDebugEnabled && ctx.Terms.Count > 1)
            {
                Log.DebugFormat("expanded {0}: {1}",
                    ctx.Query.Value, string.Join(" ", ctx.Terms.Select(t => t.Word.Value)));
            }

            Log.DebugFormat("scanned {0} in {1}", ctx.Query.Serialize(), time.Elapsed);
        }

        private void TermScan(QueryContext ctx)
        {
            IList<Term> terms;

            using (var reader = GetTreeReader(ctx.Query.Field))
            {
                if (ctx.Query.Fuzzy)
                {
                    terms = reader.SemanticallyNear(ctx.Query.Value, ctx.Query.Edits(ctx.Query.Value))
                        .ToTerms(ctx.Query.Field);
                }
                else if (ctx.Query.Prefix)
                {
                    terms = reader.StartsWith(ctx.Query.Value)
                        .ToTerms(ctx.Query.Field);
                }
                else
                {
                    terms = reader.IsWord(ctx.Query.Value)
                        .ToTerms(ctx.Query.Field);
                }
            }
            ctx.Terms = terms;
            ctx.Postings = GetPostings(ctx);
        }

        private void PhraseScan(QueryContext ctx)
        {
            var tokens = ((PhraseQuery)ctx.Query).Values;
            var postingsMatrix = new IList<DocumentPosting>[tokens.Count];

            for (int index = 0;index < tokens.Count; index++)
            {
                var token = tokens[index];
                IList<Term> terms;

                using (var reader = GetTreeReader(ctx.Query.Field))
                {
                    if (ctx.Query.Fuzzy)
                    {
                        terms = reader.SemanticallyNear(token, ctx.Query.Edits(token))
                            .ToTerms(ctx.Query.Field);
                    }
                    else if (ctx.Query.Prefix)
                    {
                        terms = reader.StartsWith(token)
                            .ToTerms(ctx.Query.Field);
                    }
                    else
                    {
                        terms = reader.IsWord(token)
                            .ToTerms(ctx.Query.Field);
                    }
                }

                var postings = GetPostings(terms);
                postingsMatrix[index] = postings;
            }

            ctx.Postings = postingsMatrix.Sum();
        }

        private void RangeScan(QueryContext ctx)
        {
            using (var reader = GetTreeReader(ctx.Query.Field))
            {
                ctx.Terms = reader.Range(ctx.Query.Value, ((RangeQuery)ctx.Query).ValueUpperBound)
                        .ToTerms(ctx.Query.Field);
            }

            ctx.Postings = GetPostings(ctx);
        }

        private IList<DocumentPosting> GetPostings(QueryContext query)
        {
            var time = Stopwatch.StartNew();

            var postings = GetPostings(query.Terms);

            Log.DebugFormat("read postings for {0} in {1}", query.Query.Serialize(), time.Elapsed);

            return postings;
        }

        private IList<DocumentPosting> GetPostings(IList<Term> terms)
        {
            var postings = terms.Count > 0 ? _readSession.ReadPostings(terms) : null;

            IList<DocumentPosting> reduced;

            if (postings == null)
            {
                reduced = new DocumentPosting[0];
            }
            else
            {
                reduced = postings.Sum();
            }

            return reduced;
        }

        private void Score(QueryContext ctx)
        {
            ctx.Scored = Score(ctx.Postings);
        }

        private IList<DocumentScore> Score(IList<DocumentPosting> postings)
        {
            var scores = new List<DocumentScore>(postings.Count);

            if (postings != null)
            {
                if (_scorerFactory == null)
                {
                    foreach (var posting in postings)
                    {
                        var docHash = _readSession.ReadDocHash(posting.DocumentId);

                        if (!docHash.IsObsolete)
                        {
                            scores.Add(new DocumentScore(posting.DocumentId, docHash.Hash, 0, _readSession.Version));
                        }
                    }
                }
                else
                {
                    if (postings.Any())
                    {
                        var docsWithTerm = postings.Count;
{'fraction_non_alphanumeric': 0.09183120402632598, 'fraction_numerical': 0.002632597754548974, 'mean_word_length': 1.2735433902481959, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7028240', 'n_tokens_mistral': 3372, 'n_tokens_neox': 3162, 'n_words': 651}