<commit_before>﻿using System;
using System.Configuration;
using System.Threading;
using System.Threading.Tasks;
using Exceptionless.Core;
using Exceptionless.Core.Extensions;
using Exceptionless.Core.Jobs;
using Exceptionless.Core.Migrations;
using Exceptionless.Core.Repositories;
using MongoDB.Driver;
using NLog.Fluent;

namespace Exceptionless.EventMigration {
    public class MongoCopyJob : JobBase {
        private readonly MongoDatabase _sourceMongoDatabase;
        private readonly MongoDatabase _mongoDatabase;

        public MongoCopyJob(MongoDatabase mongoDatabase) {
            _sourceMongoDatabase = GetMongoDatabase();
            _mongoDatabase = mongoDatabase;
        }

        protected override async Task<JobResult> RunInternalAsync(CancellationToken token) {
            var ip = Util.GetExternalIP();
            if (ip != null)
                Log.Info().Message("Public IP:" + ip).Write();

            CopyCollections("error", "errorstack", "errorstack.stats.day", "errorstack.stats.month", "jobhistory", "joblock", "log", "project.stats.day", "project.stats.month");
            
            Log.Info().Message("Running migrations...").Write();
            MongoMigrationChecker.EnsureLatest(Settings.Current.MongoConnectionString, new MongoUrl(Settings.Current.MongoConnectionString).DatabaseName);
            Log.Info().Message("Finished running migrations").Write();

            Log.Info().Message("Creating indexes...").Write();
            new ApplicationRepository(_mongoDatabase);
            new OrganizationRepository(_mongoDatabase);
            new ProjectRepository(_mongoDatabase);
            new TokenRepository(_mongoDatabase);
            new WebHookRepository(_mongoDatabase);
            new UserRepository(_mongoDatabase);
            Log.Info().Message("Finished creating indexes...").Write();

            return JobResult.Success;
        }

        private void CopyCollections(params string[] exclusions) {
            foreach (var collectionName in _sourceMongoDatabase.GetCollectionNames()) {
                if (collectionName.StartsWith("system") || exclusions.Contains(exclusion => String.Equals(collectionName, exclusion)))
                    continue;
                
                if (!_sourceMongoDatabase.CollectionExists(collectionName)) {
                    Log.Warn().Message("Collection \"{0}\" was not found in the source database.", collectionName).Write();
                    return;
                }

                Log.Info().Message("Copying collection: {0}", collectionName).Write();

                if (_mongoDatabase.CollectionExists(collectionName))
                    _mongoDatabase.DropCollection(collectionName);

                var source = _sourceMongoDatabase.GetCollection(collectionName);
                var target = _mongoDatabase.GetCollection(collectionName);
                target.InsertBatch(source.FindAll());

                Log.Info().Message("Finished copying collection: {0}", collectionName).Write();
            }
        }

        private MongoDatabase GetMongoDatabase() {<commit_msg>MongoCopyJob now clears the cache<commit_after>﻿using System;
using System.Configuration;
using System.Threading;
using System.Threading.Tasks;
using Exceptionless.Core;
using Exceptionless.Core.Caching;
using Exceptionless.Core.Extensions;
using Exceptionless.Core.Jobs;
using Exceptionless.Core.Migrations;
using Exceptionless.Core.Repositories;
using MongoDB.Driver;
using NLog.Fluent;

namespace Exceptionless.EventMigration {
    public class MongoCopyJob : JobBase {
        private readonly ICacheClient _cacheClient;
        private readonly MongoDatabase _sourceMongoDatabase;
        private readonly MongoDatabase _mongoDatabase;

        public MongoCopyJob(ICacheClient cacheClient, MongoDatabase mongoDatabase) {
            _cacheClient = cacheClient; 
            _sourceMongoDatabase = GetMongoDatabase();
            _mongoDatabase = mongoDatabase;
        }

        protected override async Task<JobResult> RunInternalAsync(CancellationToken token) {
            var ip = Util.GetExternalIP();
            if (ip != null)
                Log.Info().Message("Public IP:" + ip).Write();

            CopyCollections("error", "errorstack", "errorstack.stats.day", "errorstack.stats.month", "jobhistory", "joblock", "log", "project.stats.day", "project.stats.month");
            
            Log.Info().Message("Running migrations...").Write();
            MongoMigrationChecker.EnsureLatest(Settings.Current.MongoConnectionString, new MongoUrl(Settings.Current.MongoConnectionString).DatabaseName);
            Log.Info().Message("Finished running migrations").Write();

            Log.Info().Message("Creating indexes...").Write();
            new ApplicationRepository(_mongoDatabase);
            new OrganizationRepository(_mongoDatabase);
            new ProjectRepository(_mongoDatabase);
            new TokenRepository(_mongoDatabase);
            new WebHookRepository(_mongoDatabase);
            new UserRepository(_mongoDatabase);
            Log.Info().Message("Finished creating indexes...").Write();

            Log.Info().Message("Flushing redis cache").Write();
            _cacheClient.FlushAll();

            return JobResult.Success;
        }

        private void CopyCollections(params string[] exclusions) {
            foreach (var collectionName in _sourceMongoDatabase.GetCollectionNames()) {
                if (collectionName.StartsWith("system") || exclusions.Contains(exclusion => String.Equals(collectionName, exclusion)))
                    continue;
                
                if (!_sourceMongoDatabase.CollectionExists(collectionName)) {
                    Log.Warn().Message("Collection \"{0}\" was not found in the source database.", collectionName).Write();
                    return;
                }

                Log.Info().Message("Copying collection: {0}", collectionName).Write();

                if (_mongoDatabase.CollectionExists(collectionName))
                    _mongoDatabase.DropCollection(collectionName);

                var source = _sourceMongoDatabase.GetCollection(collectionName);
                var target = _mongoDatabase.GetCollection(collectionName);
                target.InsertBatch(source.FindAll());

                Log.Info().Message("Finished copying collection: {0}", collectionName).Write();
            }
        }

        private MongoDatabase GetMongoDatabase() {
{'fraction_non_alphanumeric': 0.1031929662193429, 'fraction_numerical': 0.0009254974548819991, 'mean_word_length': 2.7350230414746544, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5433773', 'n_tokens_mistral': 1559, 'n_tokens_neox': 1505, 'n_words': 308}