<commit_before>using System;
using System.Collections.Generic;
using System.Linq;
using C5;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.QueryParsers;
using Lucene.Net.Search;
using Purchasing.Core.Domain;
using Purchasing.Core.Queries;
using Version = Lucene.Net.Util.Version;
using UCDArch.Core.Utils;
using Purchasing.Web.Utility;

namespace Purchasing.Web.Services
{
    /// <summary>
    /// Searches using Lucene indexes
    /// TODO: MUST filter orders by user
    /// </summary>
    public class IndexSearchService : ISearchService
    {
        private readonly IIndexService _indexService;

        public IndexSearchService(IIndexService indexService)
        {
            _indexService = indexService;
        }

        public System.Collections.Generic.IList<SearchResults.OrderResult> SearchOrders(string searchTerm, int[] allowedIds)
        {
            var searcher = _indexService.GetIndexSearcherFor(Indexes.OrderHistory);
            try
            {
                IEnumerable<ScoreDoc> results = SearchIndex(searcher, allowedIds, searchTerm, SearchResults.OrderResult.SearchableFields);

                var orderResults = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new SearchResults.OrderResult
                    {
                        Id = int.Parse(doc.Get("orderid")),
                        Justification = doc.Get("justification"),
                        CreatedBy = doc.Get("createdby"),
                        DeliverTo = doc.Get("shipto"),
                        DeliverToEmail = doc.Get("shiptoemail"),
                        RequestNumber = doc.Get("requestnumber"),
                        DateCreated = DateTime.Parse(doc.Get("datecreated")),
                        PoNumber = doc.Get("ponumber"),
                        Approver = doc.Get("approver"),
                        AccountManager = doc.Get("accountmanager"),
                        Purchaser = doc.Get("purchaser")
                    }).ToList();


                return orderResults;
            }
            finally //The exception will be caught downstream
            {
                searcher.Close();
                searcher.Dispose();
            }

        }

        public System.Collections.Generic.IList<SearchResults.LineResult> SearchLineItems(string searchTerm, int[] allowedIds)
        {
            var searcher = _indexService.GetIndexSearcherFor(Indexes.LineItems);
            try
            {
                IEnumerable<ScoreDoc> results = SearchIndex(searcher, allowedIds, searchTerm, SearchResults.LineResult.SearchableFields);

                var lineResults = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new SearchResults.LineResult
                    {
                        OrderId = int.Parse(doc.Get("orderid")),
                        Description = doc.Get("description"),
                        Url = doc.Get("url"),
                        Notes = doc.Get("notes"),
                        CatalogNumber = doc.Get("catalognumber"),
                        CommodityId = doc.Get("commodityid"),
                        ReceivedNotes = doc.Get("receivednotes"),
                        RequestNumber = doc.Get("requestnumber"),
                        Quantity = decimal.Parse(doc.Get("quantity")),
                        Unit = doc.Get("unit")
                    }).ToList();

                return lineResults;
            }
            finally //The exception will be caught downstream
            {
                searcher.Close();
                searcher.Dispose();
            }

        }

        public System.Collections.Generic.IList<SearchResults.CustomFieldResult> SearchCustomFieldAnswers(string searchTerm, int[] allowedIds)
        {
            var searcher = _indexService.GetIndexSearcherFor(Indexes.CustomAnswers);
            try
            {
                IEnumerable<ScoreDoc> results = SearchIndex(searcher, allowedIds, searchTerm, SearchResults.CustomFieldResult.SearchableFields);

                var customFieldResults = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new SearchResults.CustomFieldResult
                    {
                        OrderId = int.Parse(doc.Get("orderid")),
                        RequestNumber = doc.Get("requestnumber"),
                        Answer = doc.Get("answer"),
                        Question = doc.Get("question")
                    }).ToList();


                return customFieldResults;
            }
            finally //The exception will be caught downstream
            {
                searcher.Close();
                searcher.Dispose();
            }
        }

        public System.Collections.Generic.IList<SearchResults.CommentResult> SearchComments(string searchTerm, int[] allowedIds)
        {
            var searcher = _indexService.GetIndexSearcherFor(Indexes.Comments);
            try
            {
                IEnumerable<ScoreDoc> results = SearchIndex(searcher, allowedIds, searchTerm, SearchResults.CommentResult.SearchableFields);

                var commentResults = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new SearchResults.CommentResult()
                    {
                        OrderId = int.Parse(doc.Get("orderid")),
                        RequestNumber = doc.Get("requestnumber"),
                        Text = doc.Get("text"),
                        CreatedBy = doc.Get("createdby"),
                        DateCreated = DateTime.Parse(doc.Get("datecreated"))
                    }).ToList();

                return commentResults;
            }
            finally //The exception will be caught downstream
            {
                searcher.Close();
                searcher.Dispose();
            }

        }

        public System.Collections.Generic.IList<IdAndName> SearchCommodities(string searchTerm)
        {
            return SearchLookupIndex(Indexes.Commodities, searchTerm);
        }

        public System.Collections.Generic.IList<IdAndName> SearchVendors(string searchTerm)
        {
            return SearchLookupIndex(Indexes.Vendors, searchTerm);
        }

        public System.Collections.Generic.IList<IdAndName> SearchAccounts(string searchTerm)
        {
            return SearchLookupIndex(Indexes.Accounts, searchTerm);
        }

        public System.Collections.Generic.IList<IdAndName> SearchBuildings(string searchTerm)
        {
            return SearchLookupIndex(Indexes.Buildings, searchTerm);
        }

        private System.Collections.Generic.IList<IdAndName> SearchLookupIndex(Indexes index, string searchTerm, int topN = 20)
        {
            if (string.IsNullOrWhiteSpace(searchTerm))
            {
                return new List<IdAndName>();
            }
            var searcher = _indexService.GetIndexSearcherFor(index);
            
            //Default to standard analyzer-- id field is tokenized into searchid non-stored field
            var analyzer = new StandardAnalyzer(Version.LUCENE_29);
            try
            {
                var termsQuery = new MultiFieldQueryParser(Version.LUCENE_29, new[] { "searchid", "name" }, analyzer).Parse(searchTerm);
                var results = searcher.Search(termsQuery, topN).ScoreDocs;

                var entities = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new IdAndName(doc.Get("id"), doc.Get("name"))).ToList();
<commit_msg>code cleanup<commit_after>using System;
using System.Collections.Generic;
using System.Linq;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.QueryParsers;
using Lucene.Net.Search;
using Purchasing.Core.Queries;
using Purchasing.Web.Utility;
using Version = Lucene.Net.Util.Version;

namespace Purchasing.Web.Services
{
    /// <summary>
    /// Searches using Lucene indexes
    /// TODO: MUST filter orders by user
    /// </summary>
    public class IndexSearchService : ISearchService
    {
        private readonly IIndexService _indexService;

        public IndexSearchService(IIndexService indexService)
        {
            _indexService = indexService;
        }

        public IList<SearchResults.OrderResult> SearchOrders(string searchTerm, int[] allowedIds)
        {
            var searcher = _indexService.GetIndexSearcherFor(Indexes.OrderHistory);
            try
            {
                IEnumerable<ScoreDoc> results = SearchIndex(searcher, allowedIds, searchTerm, SearchResults.OrderResult.SearchableFields);

                var orderResults = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new SearchResults.OrderResult
                    {
                        Id = int.Parse(doc.Get("orderid")),
                        Justification = doc.Get("justification"),
                        CreatedBy = doc.Get("createdby"),
                        DeliverTo = doc.Get("shipto"),
                        DeliverToEmail = doc.Get("shiptoemail"),
                        RequestNumber = doc.Get("requestnumber"),
                        DateCreated = DateTime.Parse(doc.Get("datecreated")),
                        PoNumber = doc.Get("ponumber"),
                        Approver = doc.Get("approver"),
                        AccountManager = doc.Get("accountmanager"),
                        Purchaser = doc.Get("purchaser")
                    }).ToList();


                return orderResults;
            }
            finally //The exception will be caught downstream
            {
                searcher.Close();
                searcher.Dispose();
            }

        }

        public IList<SearchResults.LineResult> SearchLineItems(string searchTerm, int[] allowedIds)
        {
            var searcher = _indexService.GetIndexSearcherFor(Indexes.LineItems);
            try
            {
                IEnumerable<ScoreDoc> results = SearchIndex(searcher, allowedIds, searchTerm, SearchResults.LineResult.SearchableFields);

                var lineResults = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new SearchResults.LineResult
                    {
                        OrderId = int.Parse(doc.Get("orderid")),
                        Description = doc.Get("description"),
                        Url = doc.Get("url"),
                        Notes = doc.Get("notes"),
                        CatalogNumber = doc.Get("catalognumber"),
                        CommodityId = doc.Get("commodityid"),
                        ReceivedNotes = doc.Get("receivednotes"),
                        RequestNumber = doc.Get("requestnumber"),
                        Quantity = decimal.Parse(doc.Get("quantity")),
                        Unit = doc.Get("unit")
                    }).ToList();

                return lineResults;
            }
            finally //The exception will be caught downstream
            {
                searcher.Close();
                searcher.Dispose();
            }

        }

        public IList<SearchResults.CustomFieldResult> SearchCustomFieldAnswers(string searchTerm, int[] allowedIds)
        {
            var searcher = _indexService.GetIndexSearcherFor(Indexes.CustomAnswers);
            try
            {
                IEnumerable<ScoreDoc> results = SearchIndex(searcher, allowedIds, searchTerm, SearchResults.CustomFieldResult.SearchableFields);

                var customFieldResults = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new SearchResults.CustomFieldResult
                    {
                        OrderId = int.Parse(doc.Get("orderid")),
                        RequestNumber = doc.Get("requestnumber"),
                        Answer = doc.Get("answer"),
                        Question = doc.Get("question")
                    }).ToList();


                return customFieldResults;
            }
            finally //The exception will be caught downstream
            {
                searcher.Close();
                searcher.Dispose();
            }
        }

        public IList<SearchResults.CommentResult> SearchComments(string searchTerm, int[] allowedIds)
        {
            var searcher = _indexService.GetIndexSearcherFor(Indexes.Comments);
            try
            {
                IEnumerable<ScoreDoc> results = SearchIndex(searcher, allowedIds, searchTerm, SearchResults.CommentResult.SearchableFields);

                var commentResults = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new SearchResults.CommentResult()
                    {
                        OrderId = int.Parse(doc.Get("orderid")),
                        RequestNumber = doc.Get("requestnumber"),
                        Text = doc.Get("text"),
                        CreatedBy = doc.Get("createdby"),
                        DateCreated = DateTime.Parse(doc.Get("datecreated"))
                    }).ToList();

                return commentResults;
            }
            finally //The exception will be caught downstream
            {
                searcher.Close();
                searcher.Dispose();
            }

        }

        public IList<IdAndName> SearchCommodities(string searchTerm)
        {
            return SearchLookupIndex(Indexes.Commodities, searchTerm);
        }

        public IList<IdAndName> SearchVendors(string searchTerm)
        {
            return SearchLookupIndex(Indexes.Vendors, searchTerm);
        }

        public IList<IdAndName> SearchAccounts(string searchTerm)
        {
            return SearchLookupIndex(Indexes.Accounts, searchTerm);
        }

        public IList<IdAndName> SearchBuildings(string searchTerm)
        {
            return SearchLookupIndex(Indexes.Buildings, searchTerm);
        }

        private IList<IdAndName> SearchLookupIndex(Indexes index, string searchTerm, int topN = 20)
        {
            if (string.IsNullOrWhiteSpace(searchTerm))
            {
                return new List<IdAndName>();
            }
            var searcher = _indexService.GetIndexSearcherFor(index);
            
            //Default to standard analyzer-- id field is tokenized into searchid non-stored field
            var analyzer = new StandardAnalyzer(Version.LUCENE_29);
            try
            {
                var termsQuery = new MultiFieldQueryParser(Version.LUCENE_29, new[] { "searchid", "name" }, analyzer).Parse(searchTerm);
                var results = searcher.Search(termsQuery, topN).ScoreDocs;

                var entities = results
                    .Select(scoreDoc => searcher.Doc(scoreDoc.doc))
                    .Select(doc => new IdAndName(doc.Get("id"), doc.Get("name"))).ToList();

{'fraction_non_alphanumeric': 0.0928217002977175, 'fraction_numerical': 0.0008600727753886868, 'mean_word_length': 1.628412450008694, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1360288', 'n_tokens_mistral': 3732, 'n_tokens_neox': 3555, 'n_words': 675}