<commit_before>﻿using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Exceptionless.Core.Configuration;
using Exceptionless.Core.Extensions;
using Exceptionless.Core.Models;
using Exceptionless.Core.Models.Data;
using Exceptionless.Core.Repositories.Queries;
using Foundatio.Parsers.ElasticQueries;
using Foundatio.Parsers.ElasticQueries.Extensions;
using Foundatio.Repositories.Elasticsearch.Configuration;
using Foundatio.Repositories.Elasticsearch.Extensions;
using Microsoft.Extensions.Logging;
using Nest;

namespace Exceptionless.Core.Repositories.Configuration {
    public sealed class EventIndex : DailyIndex<PersistentEvent> {
        private const string ALL_WORDS_DELIMITER_TOKEN_FILTER = "all_word_delimiter";
        private const string EDGE_NGRAM_TOKEN_FILTER = "edge_ngram";
        private const string EMAIL_TOKEN_FILTER = "email";
        private const string TYPENAME_TOKEN_FILTER = "typename";
        private const string VERSION_TOKEN_FILTER = "version";
        private const string VERSION_PAD1_TOKEN_FILTER = "version_pad1";
        private const string VERSION_PAD2_TOKEN_FILTER = "version_pad2";
        private const string VERSION_PAD3_TOKEN_FILTER = "version_pad3";
        private const string VERSION_PAD4_TOKEN_FILTER = "version_pad4";
        private const string TLD_STOPWORDS_TOKEN_FILTER = "tld_stopwords";

        internal const string COMMA_WHITESPACE_ANALYZER = "comma_whitespace";
        internal const string EMAIL_ANALYZER = "email";
        internal const string VERSION_INDEX_ANALYZER = "version_index";
        internal const string VERSION_SEARCH_ANALYZER = "version_search";
        internal const string WHITESPACE_LOWERCASE_ANALYZER = "whitespace_lower";
        internal const string TYPENAME_ANALYZER = "typename";
        internal const string STANDARDPLUS_ANALYZER = "standardplus";

        private const string COMMA_WHITESPACE_TOKENIZER = "comma_whitespace";
        private const string TYPENAME_HIERARCHY_TOKENIZER = "typename_hierarchy";
        
        internal const string ALL_FIELD = "all";
        
        private readonly ExceptionlessElasticConfiguration _configuration;

        public EventIndex(ExceptionlessElasticConfiguration configuration) : base(configuration, configuration.Options.ScopePrefix + "events", 1, doc => ((PersistentEvent)doc).Date.UtcDateTime) {
            _configuration = configuration;
            MaxIndexAge = TimeSpan.FromDays(180);

            AddAlias($"{Name}-today", TimeSpan.FromDays(1));
            AddAlias($"{Name}-last3days", TimeSpan.FromDays(7));
            AddAlias($"{Name}-last7days", TimeSpan.FromDays(7));
            AddAlias($"{Name}-last30days", TimeSpan.FromDays(30));
            AddAlias($"{Name}-last90days", TimeSpan.FromDays(90));
        }

        public override async Task ConfigureAsync() {
            const string FLATTEN_ERRORS_SCRIPT = @"
if (!ctx.containsKey('data') || !(ctx.data.containsKey('@error') || ctx.data.containsKey('@simple_error')))
  return;

def types = [];
def messages = [];
def codes = [];
def err = ctx.data.containsKey('@error') ? ctx.data['@error'] : ctx.data['@simple_error'];
def curr = err;
while (curr != null) {<commit_msg>REVIEW - only discard event indexes if max retention days is set, defaults in configuration to 180 days.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Exceptionless.Core.Configuration;
using Exceptionless.Core.Extensions;
using Exceptionless.Core.Models;
using Exceptionless.Core.Models.Data;
using Exceptionless.Core.Repositories.Queries;
using Foundatio.Parsers.ElasticQueries;
using Foundatio.Parsers.ElasticQueries.Extensions;
using Foundatio.Repositories.Elasticsearch.Configuration;
using Foundatio.Repositories.Elasticsearch.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Nest;

namespace Exceptionless.Core.Repositories.Configuration {
    public sealed class EventIndex : DailyIndex<PersistentEvent> {
        private const string ALL_WORDS_DELIMITER_TOKEN_FILTER = "all_word_delimiter";
        private const string EDGE_NGRAM_TOKEN_FILTER = "edge_ngram";
        private const string EMAIL_TOKEN_FILTER = "email";
        private const string TYPENAME_TOKEN_FILTER = "typename";
        private const string VERSION_TOKEN_FILTER = "version";
        private const string VERSION_PAD1_TOKEN_FILTER = "version_pad1";
        private const string VERSION_PAD2_TOKEN_FILTER = "version_pad2";
        private const string VERSION_PAD3_TOKEN_FILTER = "version_pad3";
        private const string VERSION_PAD4_TOKEN_FILTER = "version_pad4";
        private const string TLD_STOPWORDS_TOKEN_FILTER = "tld_stopwords";

        internal const string COMMA_WHITESPACE_ANALYZER = "comma_whitespace";
        internal const string EMAIL_ANALYZER = "email";
        internal const string VERSION_INDEX_ANALYZER = "version_index";
        internal const string VERSION_SEARCH_ANALYZER = "version_search";
        internal const string WHITESPACE_LOWERCASE_ANALYZER = "whitespace_lower";
        internal const string TYPENAME_ANALYZER = "typename";
        internal const string STANDARDPLUS_ANALYZER = "standardplus";

        private const string COMMA_WHITESPACE_TOKENIZER = "comma_whitespace";
        private const string TYPENAME_HIERARCHY_TOKENIZER = "typename_hierarchy";
        
        internal const string ALL_FIELD = "all";
        
        private readonly ExceptionlessElasticConfiguration _configuration;

        public EventIndex(ExceptionlessElasticConfiguration configuration, IOptions<AppOptions> options) : base(configuration, configuration.Options.ScopePrefix + "events", 1, doc => ((PersistentEvent)doc).Date.UtcDateTime) {
            _configuration = configuration;

            if (options.Value.MaximumRetentionDays > 0)
                MaxIndexAge = TimeSpan.FromDays(options.Value.MaximumRetentionDays);

            AddAlias($"{Name}-today", TimeSpan.FromDays(1));
            AddAlias($"{Name}-last3days", TimeSpan.FromDays(7));
            AddAlias($"{Name}-last7days", TimeSpan.FromDays(7));
            AddAlias($"{Name}-last30days", TimeSpan.FromDays(30));
            AddAlias($"{Name}-last90days", TimeSpan.FromDays(90));
        }

        public override async Task ConfigureAsync() {
            const string FLATTEN_ERRORS_SCRIPT = @"
if (!ctx.containsKey('data') || !(ctx.data.containsKey('@error') || ctx.data.containsKey('@simple_error')))
  return;

def types = [];
def messages = [];
def codes = [];
def err = ctx.data.containsKey('@error') ? ctx.data['@error'] : ctx.data['@simple_error'];
def curr = err;
while (curr != null) {
{'fraction_non_alphanumeric': 0.09481082697771796, 'fraction_numerical': 0.00762673844773441, 'mean_word_length': 4.237274862960063, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7661680', 'n_tokens_mistral': 1959, 'n_tokens_neox': 1911, 'n_words': 407}