<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using JetBrains.Annotations;
using Orchard.ContentManagement;
using Orchard.ContentManagement.Handlers;
using Orchard.Data;
using Orchard.Indexing;
using Orchard.Logging;
using Orchard.Services;
using Orchard.Tasks;
using Orchard.Core.Indexing.Models;
using Orchard.Tasks.Indexing;
using Orchard.Indexing;

namespace Orchard.Core.Indexing.Services {
    /// <summary>
    /// Contains the logic which is regularly executed to retrieve index information from multiple content handlers.
    /// </summary>
    [UsedImplicitly]
    public class IndexingTaskExecutor : IBackgroundTask, IIndexNotifierHandler {
        private readonly IClock _clock;
        private readonly IRepository<IndexingTaskRecord> _repository;
        private readonly IEnumerable<IContentHandler> _handlers;
        private IIndexProvider _indexProvider;
        private readonly IIndexManager _indexManager;
        private readonly IIndexingTaskManager _indexingTaskManager;
        private readonly IContentManager _contentManager;
        private const string SearchIndexName = "Search";
        
        private readonly object _synLock = new object();

        public IndexingTaskExecutor(
            IClock clock,
            IRepository<IndexingTaskRecord> repository,
            IEnumerable<IContentHandler> handlers,
            IIndexManager indexManager,
            IIndexingTaskManager indexingTaskManager,
            IContentManager contentManager) {
            _clock = clock;
            _repository = repository;
            _indexManager = indexManager;
            _handlers = handlers;
            _indexingTaskManager = indexingTaskManager;
            _contentManager = contentManager;
            Logger = NullLogger.Instance;
        }

        public ILogger Logger { get; set; }

        public void UpdateIndex(string indexName) {
            if (indexName == SearchIndexName) {
                Sweep();
            }
        }

        public void Sweep() {

            if ( !System.Threading.Monitor.TryEnter(_synLock) ) {
                Logger.Information("Index was requested but was already running");
                return;
            }

            try {

                if (!_indexManager.HasIndexProvider()) {
                    return;
                }

                _indexProvider = _indexManager.GetSearchIndexProvider();
                var updateIndexDocuments = new List<IIndexDocument>();
                var lastIndexing = DateTime.UtcNow;

                // Do we need to rebuild the full index (first time module is used, or rebuild index requested) ?
                if (_indexProvider.IsEmpty(SearchIndexName)) {
                    Logger.Information("Rebuild index started");

                    // mark current last task, as we should process older ones (in case of rebuild index only)
                    lastIndexing = _indexingTaskManager.GetLastTaskDateTime();

                    // get every existing content item to index it
                    foreach (var contentItem in _contentManager.Query(VersionOptions.Published).List()) {
                        try {
                            var context = new IndexContentContext {
                                ContentItem = contentItem,
                                IndexDocument = _indexProvider.New(contentItem.Id)
                            };

                            // dispatch to handlers to retrieve index information
                            foreach (var handler in _handlers) {
                                handler.Indexing(context);
                            }

                            if ( context.IndexDocument.IsDirty ) {
                                updateIndexDocuments.Add(context.IndexDocument);

                                foreach ( var handler in _handlers ) {
                                    handler.Indexed(context);
                                }
                            }
                        }
                        catch (Exception ex) {
                            Logger.Warning(ex, "Unable to index content item #{0} during rebuild", contentItem.Id);
                        }
                    }

                }
                else {
                    // retrieve last processed index time
                    lastIndexing = _indexProvider.GetLastIndexUtc(SearchIndexName);
                }

                _indexProvider.SetLastIndexUtc(SearchIndexName, _clock.UtcNow);

                // retrieve not yet processed tasks
                var taskRecords = _repository.Fetch(x => x.CreatedUtc >= lastIndexing)
                    .ToArray();

                if (taskRecords.Length == 0)
                    return;
<commit_msg>Corrected a bug which could cause the last content item to be indexed two times<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using JetBrains.Annotations;
using Orchard.ContentManagement;
using Orchard.ContentManagement.Handlers;
using Orchard.Data;
using Orchard.Indexing;
using Orchard.Logging;
using Orchard.Services;
using Orchard.Tasks;
using Orchard.Core.Indexing.Models;
using Orchard.Tasks.Indexing;
using Orchard.Indexing;

namespace Orchard.Core.Indexing.Services {
    /// <summary>
    /// Contains the logic which is regularly executed to retrieve index information from multiple content handlers.
    /// </summary>
    [UsedImplicitly]
    public class IndexingTaskExecutor : IBackgroundTask, IIndexNotifierHandler {
        private readonly IClock _clock;
        private readonly IRepository<IndexingTaskRecord> _repository;
        private readonly IEnumerable<IContentHandler> _handlers;
        private IIndexProvider _indexProvider;
        private readonly IIndexManager _indexManager;
        private readonly IIndexingTaskManager _indexingTaskManager;
        private readonly IContentManager _contentManager;
        private const string SearchIndexName = "Search";
        
        private readonly object _synLock = new object();

        public IndexingTaskExecutor(
            IClock clock,
            IRepository<IndexingTaskRecord> repository,
            IEnumerable<IContentHandler> handlers,
            IIndexManager indexManager,
            IIndexingTaskManager indexingTaskManager,
            IContentManager contentManager) {
            _clock = clock;
            _repository = repository;
            _indexManager = indexManager;
            _handlers = handlers;
            _indexingTaskManager = indexingTaskManager;
            _contentManager = contentManager;
            Logger = NullLogger.Instance;
        }

        public ILogger Logger { get; set; }

        public void UpdateIndex(string indexName) {
            if (indexName == SearchIndexName) {
                Sweep();
            }
        }

        public void Sweep() {

            if ( !System.Threading.Monitor.TryEnter(_synLock) ) {
                Logger.Information("Index was requested but was already running");
                return;
            }

            try {

                if (!_indexManager.HasIndexProvider()) {
                    return;
                }

                _indexProvider = _indexManager.GetSearchIndexProvider();
                var updateIndexDocuments = new List<IIndexDocument>();
                var lastIndexing = DateTime.UtcNow;

                // Do we need to rebuild the full index (first time module is used, or rebuild index requested) ?
                if (_indexProvider.IsEmpty(SearchIndexName)) {
                    Logger.Information("Rebuild index started");

                    // mark current last task, as we should process older ones (in case of rebuild index only)
                    lastIndexing = _indexingTaskManager.GetLastTaskDateTime();

                    // get every existing content item to index it
                    foreach (var contentItem in _contentManager.Query(VersionOptions.Published).List()) {
                        try {
                            var context = new IndexContentContext {
                                ContentItem = contentItem,
                                IndexDocument = _indexProvider.New(contentItem.Id)
                            };

                            // dispatch to handlers to retrieve index information
                            foreach (var handler in _handlers) {
                                handler.Indexing(context);
                            }

                            if ( context.IndexDocument.IsDirty ) {
                                updateIndexDocuments.Add(context.IndexDocument);

                                foreach ( var handler in _handlers ) {
                                    handler.Indexed(context);
                                }
                            }
                        }
                        catch (Exception ex) {
                            Logger.Warning(ex, "Unable to index content item #{0} during rebuild", contentItem.Id);
                        }
                    }

                }
                else {
                    // retrieve last processed index time
                    lastIndexing = _indexProvider.GetLastIndexUtc(SearchIndexName);
                }

                _indexProvider.SetLastIndexUtc(SearchIndexName, _clock.UtcNow);

                // retrieve not yet processed tasks
                var taskRecords = _repository.Fetch(x => x.CreatedUtc > lastIndexing)
                    .ToArray();

                // nothing to do ?
                if (taskRecords.Length + updateIndexDocuments.Count == 0)
                    return;

{'fraction_non_alphanumeric': 0.06295648595823475, 'fraction_numerical': 0.0004114803003806193, 'mean_word_length': 1.5959946595460615, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2559828', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2078, 'n_words': 578}