<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using Searchlight.Query;

namespace Searchlight
{
    public static class LinqExecutor
    {
        /// <summary>
        /// Run the query represented by this syntax tree against an in-memory collection using LINQ
        /// </summary>
        /// <param name="tree">The syntax tree of the query to execute</param>
        /// <param name="collection">The collection of data elements to query</param>
        /// <typeparam name="T">Generic type of the model</typeparam>
        /// <returns></returns>
        public static FetchResult<T> QueryCollection<T>(this SyntaxTree tree, IEnumerable<T> collection)
        {
            // Goal of this function is to construct this LINQ expression:
            //   return (from obj in LIST where QUERY select obj)
            // Here's how we'll do it:
            var queryable = collection.AsQueryable<T>();
            
            // If the user specified a filter 
            var select = Expression.Parameter(typeof(T), "obj");
            var expression = BuildExpression(select, tree.Filter, tree.Source);
            if (expression != null)
            {
                var whereCallExpression = Expression.Call(
                    typeof(Queryable),
                    "Where",
                    new Type[] { queryable.ElementType },
                    queryable.Expression,
                    Expression.Lambda<Func<T, bool>>(expression, new ParameterExpression[] { select }));
                queryable = queryable.Provider.CreateQuery<T>(whereCallExpression);
            }

            // If the user specified a sorting clause
            if (tree.OrderBy.Any())
            {
                queryable = InternalOrderBy<T>(queryable, tree.OrderBy);
            }

            // Compute the list once and keep track of full length
            var filteredAndSorted = queryable.ToList();
            var totalCount = filteredAndSorted.Count;
            
            // If the user requested pagination
            IEnumerable<T> paginated = filteredAndSorted;
            if (tree.PageNumber > 0 && tree.PageSize > 0)
            {
                // case 1: user specified page number and page size
                paginated = filteredAndSorted.Skip((int)(tree.PageSize * tree.PageNumber)).Take((int)tree.PageSize);
            }
            else if (tree.PageNumber == 0 && tree.PageSize > 0)
            {
                // case 2: user specified a page size but no page number
                paginated = filteredAndSorted.Take((int)tree.PageSize);
            }

            // construct the return fetch result
            var result = new FetchResult<T>
            {
                pageSize = tree.PageSize,
                pageNumber = tree.PageNumber,
                totalCount = totalCount,
                records = paginated.ToArray()
            };

            return result;
        }
        
        internal static IQueryable<T> InternalOrderBy<T>(IQueryable source, List<SortInfo> orderBy)
        {
            var queryExpr = source.Expression;
            int count = 0;
            ParameterExpression[] parameterExpressions =
            {
                Expression.Parameter(source.ElementType, "param0")
            };
            foreach (var sort in orderBy)
            {
                var methodName = count == 0 ? "OrderBy" : "ThenBy";
                if (sort.Direction == SortDirection.Descending)
                {
                    methodName += "Descending";
                }
                var field = Expression.Property(Expression.Parameter(typeof(T)), sort.Column.FieldName);
                var quote = Expression.Quote(Expression.Lambda(field, parameterExpressions));
                queryExpr = Expression.Call(
                    typeof(Queryable), 
                    methodName,
                    new[] { source.ElementType, typeof(T) },
                    queryExpr, 
                    quote);
                count++;
            }

            return source.Provider.CreateQuery<T>(queryExpr);
        }

        /// <summary>
        /// Build a complex expression from a list of clauses
        /// </summary>
        /// <param name="select"></param>
        /// <param name="query"></param>
        /// <param name="src"></param>
        /// <returns></returns>
        private static Expression BuildExpression(ParameterExpression select, List<BaseClause> query, DataSource src)
        {
            ConjunctionType ct = ConjunctionType.NONE;
            Expression result = null;
            foreach (var clause in query)
            {
                var clauseExpression = BuildOneExpression(select, clause, src);

                // First clause starts a run
                if (result == null)<commit_msg>Wrong field type!  Fixing that<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using Searchlight.Query;

namespace Searchlight
{
    public static class LinqExecutor
    {
        /// <summary>
        /// Run the query represented by this syntax tree against an in-memory collection using LINQ
        /// </summary>
        /// <param name="tree">The syntax tree of the query to execute</param>
        /// <param name="collection">The collection of data elements to query</param>
        /// <typeparam name="T">Generic type of the model</typeparam>
        /// <returns></returns>
        public static FetchResult<T> QueryCollection<T>(this SyntaxTree tree, IEnumerable<T> collection)
        {
            // Goal of this function is to construct this LINQ expression:
            //   return (from obj in LIST where QUERY select obj)
            // Here's how we'll do it:
            var queryable = collection.AsQueryable<T>();
            
            // If the user specified a filter 
            var select = Expression.Parameter(typeof(T), "obj");
            var expression = BuildExpression(select, tree.Filter, tree.Source);
            if (expression != null)
            {
                var whereCallExpression = Expression.Call(
                    typeof(Queryable),
                    "Where",
                    new Type[] { queryable.ElementType },
                    queryable.Expression,
                    Expression.Lambda<Func<T, bool>>(expression, new ParameterExpression[] { select }));
                queryable = queryable.Provider.CreateQuery<T>(whereCallExpression);
            }

            // If the user specified a sorting clause
            if (tree.OrderBy.Any())
            {
                queryable = InternalOrderBy<T>(queryable, tree.OrderBy);
            }

            // Compute the list once and keep track of full length
            var filteredAndSorted = queryable.ToList();
            var totalCount = filteredAndSorted.Count;
            
            // If the user requested pagination
            IEnumerable<T> paginated = filteredAndSorted;
            if (tree.PageNumber > 0 && tree.PageSize > 0)
            {
                // case 1: user specified page number and page size
                paginated = filteredAndSorted.Skip((int)(tree.PageSize * tree.PageNumber)).Take((int)tree.PageSize);
            }
            else if (tree.PageNumber == 0 && tree.PageSize > 0)
            {
                // case 2: user specified a page size but no page number
                paginated = filteredAndSorted.Take((int)tree.PageSize);
            }

            // construct the return fetch result
            var result = new FetchResult<T>
            {
                pageSize = tree.PageSize,
                pageNumber = tree.PageNumber,
                totalCount = totalCount,
                records = paginated.ToArray()
            };

            return result;
        }
        
        internal static IQueryable<T> InternalOrderBy<T>(IQueryable source, List<SortInfo> orderBy)
        {
            var queryExpr = source.Expression;
            int count = 0;
            ParameterExpression[] parameterExpressions =
            {
                Expression.Parameter(source.ElementType, "param0")
            };
            foreach (var sort in orderBy)
            {
                var methodName = count == 0 ? "OrderBy" : "ThenBy";
                if (sort.Direction == SortDirection.Descending)
                {
                    methodName += "Descending";
                }
                var field = Expression.Property(Expression.Parameter(typeof(T)), sort.Column.FieldName);
                var quote = Expression.Quote(Expression.Lambda(field, parameterExpressions));
                queryExpr = Expression.Call(
                    typeof(Queryable), 
                    methodName,
                    new[] { source.ElementType, sort.Column.FieldType },
                    queryExpr, 
                    quote);
                count++;
            }

            return source.Provider.CreateQuery<T>(queryExpr);
        }

        /// <summary>
        /// Build a complex expression from a list of clauses
        /// </summary>
        /// <param name="select"></param>
        /// <param name="query"></param>
        /// <param name="src"></param>
        /// <returns></returns>
        private static Expression BuildExpression(ParameterExpression select, List<BaseClause> query, DataSource src)
        {
            ConjunctionType ct = ConjunctionType.NONE;
            Expression result = null;
            foreach (var clause in query)
            {
                var clauseExpression = BuildOneExpression(select, clause, src);

                // First clause starts a run
                if (result == null)
{'fraction_non_alphanumeric': 0.0955257499744036, 'fraction_numerical': 0.0018429405139756322, 'mean_word_length': 1.597872340425532, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3928776', 'n_tokens_mistral': 2363, 'n_tokens_neox': 2241, 'n_words': 643}