<commit_before>﻿using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Simple.Data.Ado;

namespace Simple.Data.SqlServer
{
    [Export(typeof(IQueryPager))]
    public class SqlQueryPager : IQueryPager
    {
        private static readonly Regex ColumnExtract = new Regex(@"SELECT\s*(.*)\s*(\sFROM.*)", RegexOptions.Multiline | RegexOptions.IgnoreCase);
        private static readonly Regex SelectMatch = new Regex(@"^SELECT\s*(DISTINCT)?", RegexOptions.IgnoreCase);

        public IEnumerable<string> ApplyLimit(string sql, int take)
        {
            yield return SelectMatch.Replace(sql, match => match.Value + " TOP " + take + " ");
        }

        public IEnumerable<string> ApplyPaging(string sql, string[] keys, int skip, int take)
        {
            sql = sql.Replace(Environment.NewLine, " ");
            var builder = new StringBuilder("WITH __Data AS (SELECT ");

            var match = ColumnExtract.Match(sql);
            var columns = match.Groups[1].Value.Trim();
            var fromEtc = match.Groups[2].Value.Trim();

            builder.Append(string.Join(",", keys));

            var orderBy = ExtractOrderBy(columns, keys, ref fromEtc);

            builder.AppendFormat(", ROW_NUMBER() OVER({0}) AS [_#_]", orderBy);
            builder.AppendLine();
            builder.Append(fromEtc);
            builder.AppendLine(")");
            builder.AppendFormat("SELECT {0} FROM __Data ", columns);
            builder.AppendFormat("JOIN {0} ON ",
                                 keys[0].Substring(0, keys[0].LastIndexOf(".", StringComparison.OrdinalIgnoreCase)));
            if (keys.Length > 1)
                builder.AppendFormat(string.Join(" AND ", keys.Select(MakeDataJoin)));
            else
                builder.AppendFormat(string.Join(" ", keys.Select(MakeDataJoin)));
            var rest = Regex.Replace(fromEtc, @"^from (\[.*?\]\.\[.*?\])", @"");
            builder.Append(rest);
            <commit_msg>Fixes #264<commit_after>﻿using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Simple.Data.Ado;

namespace Simple.Data.SqlServer
{
    [Export(typeof(IQueryPager))]
    public class SqlQueryPager : IQueryPager
    {
        private static readonly Regex ColumnExtract = new Regex(@"SELECT\s*(.*)\s*(\sFROM.*)", RegexOptions.Multiline | RegexOptions.IgnoreCase);
        private static readonly Regex SelectMatch = new Regex(@"^SELECT\s*(DISTINCT)?", RegexOptions.IgnoreCase);

        public IEnumerable<string> ApplyLimit(string sql, int take)
        {
            yield return SelectMatch.Replace(sql, match => match.Value + " TOP " + take + " ");
        }

        public IEnumerable<string> ApplyPaging(string sql, string[] keys, int skip, int take)
        {
            sql = sql.Replace(Environment.NewLine, " ");
            var builder = new StringBuilder("WITH __Data AS (SELECT ");

            var match = ColumnExtract.Match(sql);
            var columns = match.Groups[1].Value.Trim();
            var fromEtc = match.Groups[2].Value.Trim();

            builder.Append(string.Join(",", keys));

            var orderBy = ExtractOrderBy(columns, keys, ref fromEtc);

            builder.AppendFormat(", ROW_NUMBER() OVER({0}) AS [_#_]", orderBy);
            builder.AppendLine();
            builder.Append(fromEtc);
            builder.AppendLine(")");
            builder.AppendFormat("SELECT {0} FROM __Data ", columns);
            builder.AppendFormat("JOIN {0} ON ",
                                 keys[0].Substring(0, keys[0].LastIndexOf(".", StringComparison.OrdinalIgnoreCase)));
            if (keys.Length > 1)
                builder.AppendFormat(string.Join(" AND ", keys.Select(MakeDataJoin)));
            else
                builder.AppendFormat(MakeDataJoin(keys[0]));
            var rest = Regex.Replace(fromEtc, @"^from (\[.*?\]\.\[.*?\])", @"");
            builder.Append(rest);
            
{'fraction_non_alphanumeric': 0.14012121212121212, 'fraction_numerical': 0.005333333333333333, 'mean_word_length': 2.5385934819897082, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1994778', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1126, 'n_words': 240}