<commit_before>﻿using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Web.Http;
using DotJEM.Json.Index;
using DotJEM.Json.Index.Searching;
using Lucene.Net.Search;
using Newtonsoft.Json.Linq;

namespace DotJEM.Web.Host.Controllers
{
    public class SearchController : ApiController
    {
        private readonly IStorageIndex index;

        public SearchController(IStorageIndex index)
        {
            this.index = index;
        }

        [HttpGet]
        public dynamic Get([FromUri]string query, [FromUri]int skip = 0, [FromUri]int take = 25)
        {
            if (string.IsNullOrWhiteSpace(query))
                Request.CreateErrorResponse(HttpStatusCode.BadRequest, "Must specify a query.");

            ILuceneSearcher searcher = index.CreateSearcher();
            return searcher.Search(query)
                .Skip(skip)
                .Take(take)
                .Select(hit => hit.Json);
        }

        [HttpPost]
        public dynamic Post([FromBody]dynamic value, [FromUri]string contentType = "", [FromUri]int skip = 0, [FromUri]int take = 25, [FromUri]string sort = "_Created:desc")
        {
            ILuceneSearcher searcher = index.CreateSearcher();
            ISearchResult result = searcher.Search((JObject)value, contentType).Skip(skip).Take(take).Sort(

                new Sort(sort.Split(',').Select(x =>
                {
                    string[] f = x.Split(':');
                    return new SortField(f[0], SortField.LONG, f[1].Equals("desc", StringComparison.InvariantCultureIgnoreCase));
                }).ToArray())

                );
            return result.Select(hit => hit.Json).ToArray();
        }
    }
}<commit_msg>Added total count to reduce and search results.<commit_after>﻿using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Web.Http;
using DotJEM.Json.Index;
using DotJEM.Json.Index.Searching;
using Lucene.Net.Search;
using Newtonsoft.Json.Linq;

namespace DotJEM.Web.Host.Controllers
{
    public class SearchController : ApiController
    {
        private readonly IStorageIndex index;

        public SearchController(IStorageIndex index)
        {
            this.index = index;
        }

        [HttpGet]
        public dynamic Get([FromUri]string query, [FromUri]int skip = 0, [FromUri]int take = 25)
        {
            if (string.IsNullOrWhiteSpace(query))
                Request.CreateErrorResponse(HttpStatusCode.BadRequest, "Must specify a query.");

            ILuceneSearcher searcher = index.CreateSearcher();
            ISearchResult result = searcher.Search(query);

            dynamic json = new JObject();
            json.Results = JArray.FromObject(result.Skip(skip).Take(take).Select(hit => hit.Json));
            json.TotalCount = result.TotalCount;
            return json;
        }

        [HttpPost]
        public dynamic Post([FromBody]dynamic value, [FromUri]string contentType = "", [FromUri]int skip = 0, [FromUri]int take = 25, [FromUri]string sort = "_Created:desc")
        {
            ILuceneSearcher searcher = index.CreateSearcher();
            ISearchResult result = searcher.Search((JObject)value, contentType).Skip(skip).Take(take).Sort(

                new Sort(sort.Split(',').Select(x =>
                {
                    string[] f = x.Split(':');
                    return new SortField(f[0], SortField.LONG, f[1].Equals("desc", StringComparison.InvariantCultureIgnoreCase));
                }).ToArray())

                );

            dynamic json = new JObject();
            json.Results = JArray.FromObject(result.Select(hit => hit.Json));
            json.TotalCount = result.TotalCount;
            return json;
        }
    }
}
{'fraction_non_alphanumeric': 0.11448349307774228, 'fraction_numerical': 0.004259850905218318, 'mean_word_length': 2.2697998259355963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '520897', 'n_tokens_mistral': 1064, 'n_tokens_neox': 1041, 'n_words': 219}