<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using Elasticsearch.Net;
using Exceptionless.Core.Extensions;
using Exceptionless.Core.Messaging.Models;
using Exceptionless.Core.Models;
using FluentValidation;
using Foundatio.Caching;
using Foundatio.Messaging;
using Nest;
using NLog.Fluent;

namespace Exceptionless.Core.Repositories {
    public abstract class ElasticSearchRepository<T> : ElasticSearchReadOnlyRepository<T>, IRepository<T> where T : class, IIdentity, new() {
        protected readonly IValidator<T> _validator;
        protected readonly IMessagePublisher _messagePublisher;
        protected readonly static bool _isOwnedByOrganization = typeof(IOwnedByOrganization).IsAssignableFrom(typeof(T));
        protected readonly static bool _isOwnedByProject = typeof(IOwnedByProject).IsAssignableFrom(typeof(T));
        protected readonly static bool _isOwnedByStack = typeof(IOwnedByStack).IsAssignableFrom(typeof(T));
        protected readonly static bool _hasDates = typeof(IHaveDates).IsAssignableFrom(typeof(T));
        protected readonly static bool _hasCreatedDate = typeof(IHaveCreatedDate).IsAssignableFrom(typeof(T));

        protected ElasticSearchRepository(IElasticClient elasticClient, string index = null, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, cacheClient) {
            _validator = validator;
            _messagePublisher = messagePublisher;
        }

        public bool BatchNotifications { get; set; }

        public T Add(T document, bool addToCache = false, TimeSpan? expiresIn = null, bool sendNotification = true) {
            if (document == null)
                throw new ArgumentNullException("document");

            Add(new[] { document }, addToCache, expiresIn, sendNotification);
            return document;
        }

        protected virtual void BeforeAdd(ICollection<T> documents) {
            if (_hasDates)
                documents.Cast<IHaveDates>().SetDates();
            else if (_hasCreatedDate)
                documents.Cast<IHaveCreatedDate>().SetCreatedDates();

            documents.EnsureIds();
        }

        public void Add(ICollection<T> documents, bool addToCache = false, TimeSpan? expiresIn = null, bool sendNotification = true) {
            if (documents == null || documents.Count == 0)
                throw new ArgumentException("Must provide one or more documents to add.", "documents");

            BeforeAdd(documents);

            if (_validator != null)
                documents.ForEach(_validator.ValidateAndThrow);

            if (_isEvent)
                foreach (var group in documents.Cast<PersistentEvent>().GroupBy(e => e.Date.ToUniversalTime().Date)) {
                    var result = _elasticClient.IndexMany(group.ToList(), type: "events", index: String.Concat(EventsIndexName, "-", group.Key.ToString("yyyyMM")));
                    if (!result.IsValid)
                        throw new ApplicationException(String.Join("\r\n", result.ItemsWithErrors.Select(i => i.Error)), result.ConnectionStatus.OriginalException);
                }
            else {
                var result = _elasticClient.IndexMany(documents, _index);
                if (!result.IsValid)
                    throw new ApplicationException(String.Join("\r\n", result.ItemsWithErrors.Select(i => i.Error)), result.ConnectionStatus.OriginalException);
            }

            AfterAdd(documents, addToCache, expiresIn, sendNotification);
        }

        protected virtual void AfterAdd(ICollection<T> documents, bool addToCache = false, TimeSpan? expiresIn = null, bool sendNotification = true) {
            if (!EnableCache && !sendNotification)
                return;
            
            foreach (var document in documents) {
                if (EnableCache) {
                    InvalidateCache(document);
                    if (addToCache && Cache != null)
                        Cache.Set(GetScopedCacheKey(document.Id), document, expiresIn.HasValue ? expiresIn.Value : TimeSpan.FromSeconds(RepositoryConstants.DEFAULT_CACHE_EXPIRATION_SECONDS));
                }

                if (sendNotification && !BatchNotifications)
                    PublishMessage(ChangeType.Added, document);
            }

            if (sendNotification && BatchNotifications)
                PublishMessage(ChangeType.Added, documents);
        }

        public void Remove(string id, bool sendNotification = true) {
            if (String.IsNullOrEmpty(id))
                throw new ArgumentNullException("id");

            var document = GetById(id, true);
            Remove(new[] { document }, sendNotification);
        }

        public void Remove(T document, bool sendNotification = true) {
            if (document == null)
                throw new ArgumentNullException("document");

            Remove(new[] { document }, sendNotification);
        }

        protected virtual void BeforeRemove(ICollection<T> documents) {
            if (EnableCache)
                documents.ForEach(InvalidateCache);
        }

        public void Remove(ICollection<T> documents, bool sendNotification = true) {
            if (documents == null || documents.Count == 0)
                throw new ArgumentException("Must provide one or more documents to remove.", "documents");

            BeforeRemove(documents);
            _elasticClient.DeleteByQuery<T>(q => q.Query(q1 => q1.Ids(documents.Select(d => d.Id))).Index(_index));
            AfterRemove(documents, sendNotification);
        }

        protected virtual void AfterRemove(ICollection<T> documents, bool sendNotification = true) {
            if (!EnableCache && !sendNotification)
                return;

            foreach (var document in documents) {
                if (EnableCache)
                    InvalidateCache(document);

                if (sendNotification && !BatchNotifications)
                    PublishMessage(ChangeType.Removed, document);
            }

            if (sendNotification && BatchNotifications)
                PublishMessage(ChangeType.Removed, documents);
        }

        public void RemoveAll() {
            if (EnableCache)
                Cache.FlushAll();

            if (_isEvent)
                _elasticClient.DeleteIndex(d => d.Index(String.Concat(EventsIndexName, "-*")));
            else if (_isStack)
                _elasticClient.DeleteIndex(d => d.Index(StacksIndexName));
            else
                RemoveAll(new QueryOptions(), false);

            // TODO: Update alias..
        }

        protected long RemoveAll(QueryOptions options, bool sendNotifications = true) {
            if (options == null)
                throw new ArgumentNullException("options");
<commit_msg>Updates to #110 Only drop the index for events.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using Elasticsearch.Net;
using Exceptionless.Core.Extensions;
using Exceptionless.Core.Messaging.Models;
using Exceptionless.Core.Models;
using FluentValidation;
using Foundatio.Caching;
using Foundatio.Messaging;
using Nest;
using NLog.Fluent;

namespace Exceptionless.Core.Repositories {
    public abstract class ElasticSearchRepository<T> : ElasticSearchReadOnlyRepository<T>, IRepository<T> where T : class, IIdentity, new() {
        protected readonly IValidator<T> _validator;
        protected readonly IMessagePublisher _messagePublisher;
        protected readonly static bool _isOwnedByOrganization = typeof(IOwnedByOrganization).IsAssignableFrom(typeof(T));
        protected readonly static bool _isOwnedByProject = typeof(IOwnedByProject).IsAssignableFrom(typeof(T));
        protected readonly static bool _isOwnedByStack = typeof(IOwnedByStack).IsAssignableFrom(typeof(T));
        protected readonly static bool _hasDates = typeof(IHaveDates).IsAssignableFrom(typeof(T));
        protected readonly static bool _hasCreatedDate = typeof(IHaveCreatedDate).IsAssignableFrom(typeof(T));

        protected ElasticSearchRepository(IElasticClient elasticClient, string index = null, IValidator<T> validator = null, ICacheClient cacheClient = null, IMessagePublisher messagePublisher = null) : base(elasticClient, index, cacheClient) {
            _validator = validator;
            _messagePublisher = messagePublisher;
        }

        public bool BatchNotifications { get; set; }

        public T Add(T document, bool addToCache = false, TimeSpan? expiresIn = null, bool sendNotification = true) {
            if (document == null)
                throw new ArgumentNullException("document");

            Add(new[] { document }, addToCache, expiresIn, sendNotification);
            return document;
        }

        protected virtual void BeforeAdd(ICollection<T> documents) {
            if (_hasDates)
                documents.Cast<IHaveDates>().SetDates();
            else if (_hasCreatedDate)
                documents.Cast<IHaveCreatedDate>().SetCreatedDates();

            documents.EnsureIds();
        }

        public void Add(ICollection<T> documents, bool addToCache = false, TimeSpan? expiresIn = null, bool sendNotification = true) {
            if (documents == null || documents.Count == 0)
                throw new ArgumentException("Must provide one or more documents to add.", "documents");

            BeforeAdd(documents);

            if (_validator != null)
                documents.ForEach(_validator.ValidateAndThrow);

            if (_isEvent)
                foreach (var group in documents.Cast<PersistentEvent>().GroupBy(e => e.Date.ToUniversalTime().Date)) {
                    var result = _elasticClient.IndexMany(group.ToList(), type: "events", index: String.Concat(EventsIndexName, "-", group.Key.ToString("yyyyMM")));
                    if (!result.IsValid)
                        throw new ApplicationException(String.Join("\r\n", result.ItemsWithErrors.Select(i => i.Error)), result.ConnectionStatus.OriginalException);
                }
            else {
                var result = _elasticClient.IndexMany(documents, _index);
                if (!result.IsValid)
                    throw new ApplicationException(String.Join("\r\n", result.ItemsWithErrors.Select(i => i.Error)), result.ConnectionStatus.OriginalException);
            }

            AfterAdd(documents, addToCache, expiresIn, sendNotification);
        }

        protected virtual void AfterAdd(ICollection<T> documents, bool addToCache = false, TimeSpan? expiresIn = null, bool sendNotification = true) {
            if (!EnableCache && !sendNotification)
                return;
            
            foreach (var document in documents) {
                if (EnableCache) {
                    InvalidateCache(document);
                    if (addToCache && Cache != null)
                        Cache.Set(GetScopedCacheKey(document.Id), document, expiresIn.HasValue ? expiresIn.Value : TimeSpan.FromSeconds(RepositoryConstants.DEFAULT_CACHE_EXPIRATION_SECONDS));
                }

                if (sendNotification && !BatchNotifications)
                    PublishMessage(ChangeType.Added, document);
            }

            if (sendNotification && BatchNotifications)
                PublishMessage(ChangeType.Added, documents);
        }

        public void Remove(string id, bool sendNotification = true) {
            if (String.IsNullOrEmpty(id))
                throw new ArgumentNullException("id");

            var document = GetById(id, true);
            Remove(new[] { document }, sendNotification);
        }

        public void Remove(T document, bool sendNotification = true) {
            if (document == null)
                throw new ArgumentNullException("document");

            Remove(new[] { document }, sendNotification);
        }

        protected virtual void BeforeRemove(ICollection<T> documents) {
            if (EnableCache)
                documents.ForEach(InvalidateCache);
        }

        public void Remove(ICollection<T> documents, bool sendNotification = true) {
            if (documents == null || documents.Count == 0)
                throw new ArgumentException("Must provide one or more documents to remove.", "documents");

            BeforeRemove(documents);
            _elasticClient.DeleteByQuery<T>(q => q.Query(q1 => q1.Ids(documents.Select(d => d.Id))).Index(_index));
            AfterRemove(documents, sendNotification);
        }

        protected virtual void AfterRemove(ICollection<T> documents, bool sendNotification = true) {
            if (!EnableCache && !sendNotification)
                return;

            foreach (var document in documents) {
                if (EnableCache)
                    InvalidateCache(document);

                if (sendNotification && !BatchNotifications)
                    PublishMessage(ChangeType.Removed, document);
            }

            if (sendNotification && BatchNotifications)
                PublishMessage(ChangeType.Removed, documents);
        }

        public void RemoveAll() {
            if (EnableCache)
                Cache.FlushAll();

            if (_isEvent)
                _elasticClient.DeleteIndex(d => d.Index(String.Concat(EventsIndexName, "-*")));
            else
                RemoveAll(new QueryOptions(), false);
        }

        protected long RemoveAll(QueryOptions options, bool sendNotifications = true) {
            if (options == null)
                throw new ArgumentNullException("options");

{'fraction_non_alphanumeric': 0.09381663113006397, 'fraction_numerical': 0.0008087640614660687, 'mean_word_length': 2.444416307926057, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3518543', 'n_tokens_mistral': 3410, 'n_tokens_neox': 3232, 'n_words': 798}