<commit_before>﻿using Elasticsearch.Net;
using Elasticsearch.Net.Connection;
using FakeItEasy;
using NUnit.Framework;
using OneCog.Spark.Sparkles.Document;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace OneCog.Spark.Sparkles.Tests
{
    [TestFixture]
    public class ElasticSinkTestFixture
    {
        private static Configuration.Index _tempIndex;
        private static Configuration.Index _lightIndex;
        private static Configuration.Index _humidityIndex;

        private Configuration.IElasticSearch _elasticSearchSettings;
        private Configuration.ISettings _settings;
        private IClock _clock;

        [TestFixtureSetUp]
        public static void TestSetup()
        {
            _tempIndex = new Configuration.Index { Name = "tempIndex", AppendDate = true, DateFormat = "yyy.MM.dd" };
            _lightIndex = new Configuration.Index { Name = "lightIndex", AppendDate = true };
            _humidityIndex = new Configuration.Index { Name = "humidityIndex", AppendDate = false };
        }

        [SetUp]
        public void Setup()
        {
            _elasticSearchSettings = A.Fake<Configuration.IElasticSearch>();
            A.CallTo(() => _elasticSearchSettings.Host).Returns("http://localhost:9220");
            A.CallTo(() => _elasticSearchSettings.Indexes).Returns(new[] { _tempIndex, _humidityIndex, _lightIndex });

            _settings = A.Fake<Configuration.ISettings>();
            A.CallTo(() => _settings.SparkCore).Returns(A.Fake<Configuration.ISparkCore>());
            A.CallTo(() => _settings.ElasticSearch).Returns(_elasticSearchSettings);

            _clock = A.Fake<IClock>();
            A.CallTo(() => _clock.UtcNow).Returns(DateTime.UtcNow);
        }

        [Test]
        public void ShouldIndexDocument()
        {
            IElasticClient elasticClient = A.Fake<IElasticClient>();

            IDocument document = A.Fake<IDocument>();
            A.CallTo(() => document.IndexName).Returns("humidityIndex");
            A.CallTo(() => document.Type).Returns("documentType");
            A.CallTo(() => document.Body).Returns("documentBody");

            A.CallTo(() => elasticClient.Index("humidityIndex", "documentType", "documentBody")).Returns(Fallible.Success("Test"));

            ElasticSink subject = new ElasticSink(_settings, elasticClient, _clock);

            subject.OnNext(document);

            A.CallTo(() => elasticClient.Index("humidityIndex", "documentType", "documentBody")).MustHaveHappened(Repeated.Exactly.Once);
        }

        [Test]
        public void ShouldUseIndexersToDetermineIndex()
        {
            IElasticClient elasticClient = A.Fake<IElasticClient>();

            DateTime indexDate = DateTime.UtcNow;
            A.CallTo(() => _clock.UtcNow).Returns(indexDate);

            string index = string.Format("{0}-{1}", "tempIndex", indexDate.ToString("yyyy.MM.dd"));

            IDocument document = A.Fake<IDocument>();
            A.CallTo(() => document.IndexName).Returns("tempIndex");
            A.CallTo(() => document.Type).Returns("documentType");
            A.CallTo(() => document.Body).Returns("documentBody");

            A.CallTo(() => elasticClient.Index(index, "documentType", "documentBody")).Returns(Fallible.Success("Test"));

            ElasticSink subject = new ElasticSink(_settings, elasticClient, _clock);

            subject.OnNext(document);

            A.CallTo(() => elasticClient.Index(index, "documentType", "documentBody")).MustHaveHappened(Repeated.Exactly.Once);
        }
    }
}
<commit_msg>Additional test fixtures<commit_after>﻿using Elasticsearch.Net;
using Elasticsearch.Net.Connection;
using FakeItEasy;
using NUnit.Framework;
using OneCog.Spark.Sparkles.Document;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace OneCog.Spark.Sparkles.Tests
{
    [TestFixture]
    public class ElasticSinkTestFixture
    {
        private static Configuration.Index _tempIndex;
        private static Configuration.Index _lightIndex;
        private static Configuration.Index _humidityIndex;

        private Configuration.IElasticSearch _elasticSearchSettings;
        private Configuration.ISettings _settings;
        private IClock _clock;

        [TestFixtureSetUp]
        public static void TestSetup()
        {
            _tempIndex = new Configuration.Index { Name = "tempIndex", AppendDate = true, DateFormat = "yyy.MM.dd" };
            _lightIndex = new Configuration.Index { Name = "lightIndex", AppendDate = true };
            _humidityIndex = new Configuration.Index { Name = "humidityIndex", AppendDate = false };
        }

        [SetUp]
        public void Setup()
        {
            _elasticSearchSettings = A.Fake<Configuration.IElasticSearch>();
            A.CallTo(() => _elasticSearchSettings.Host).Returns("http://localhost:9220");
            A.CallTo(() => _elasticSearchSettings.Indexes).Returns(new[] { _tempIndex, _humidityIndex, _lightIndex });

            _settings = A.Fake<Configuration.ISettings>();
            A.CallTo(() => _settings.SparkCore).Returns(A.Fake<Configuration.ISparkCore>());
            A.CallTo(() => _settings.ElasticSearch).Returns(_elasticSearchSettings);

            _clock = A.Fake<IClock>();
            A.CallTo(() => _clock.UtcNow).Returns(DateTime.UtcNow);
        }

        [Test]
        public void ShouldIndexDocument()
        {
            IElasticClient elasticClient = A.Fake<IElasticClient>();

            IDocument document = A.Fake<IDocument>();
            A.CallTo(() => document.IndexName).Returns("humidityIndex");
            A.CallTo(() => document.Type).Returns("documentType");
            A.CallTo(() => document.Body).Returns("documentBody");

            A.CallTo(() => elasticClient.Index("humidityIndex", "documentType", "documentBody")).Returns(Fallible.Success("Test"));

            ElasticSink subject = new ElasticSink(_settings, elasticClient, _clock);

            subject.OnNext(document);

            A.CallTo(() => elasticClient.Index("humidityIndex", "documentType", "documentBody")).MustHaveHappened(Repeated.Exactly.Once);
        }

        [Test]
        public void ShouldUseIndexersToDetermineIndex()
        {
            IElasticClient elasticClient = A.Fake<IElasticClient>();

            DateTime indexDate = DateTime.UtcNow;
            A.CallTo(() => _clock.UtcNow).Returns(indexDate);

            string index = string.Format("{0}-{1}", "tempIndex", indexDate.ToString("yyyy.MM.dd"));

            IDocument document = A.Fake<IDocument>();
            A.CallTo(() => document.IndexName).Returns("tempIndex");
            A.CallTo(() => document.Type).Returns("documentType");
            A.CallTo(() => document.Body).Returns("documentBody");

            A.CallTo(() => elasticClient.Index(index, "documentType", "documentBody")).Returns(Fallible.Success("Test"));

            ElasticSink subject = new ElasticSink(_settings, elasticClient, _clock);

            subject.OnNext(document);

            A.CallTo(() => elasticClient.Index(index, "documentType", "documentBody")).MustHaveHappened(Repeated.Exactly.Once);
        }

        [Test]
        public void ShouldContiueToIndexIfIndexerCannotBeFound()
        {
            IElasticClient elasticClient = A.Fake<IElasticClient>();

            DateTime indexDate = DateTime.UtcNow;
            A.CallTo(() => _clock.UtcNow).Returns(indexDate);

            string index = string.Format("{0}-{1}", "tempIndex", indexDate.ToString("yyyy.MM.dd"));

            IDocument document = A.Fake<IDocument>();
            A.CallTo(() => document.IndexName).Returns("missingIndex");
            A.CallTo(() => document.Type).Returns("documentType");
            A.CallTo(() => document.Body).Returns("documentBody");

            A.CallTo(() => elasticClient.Index(index, "documentType", "documentBody")).Returns(Fallible.Success("Test"));

            ElasticSink subject = new ElasticSink(_settings, elasticClient, _clock);

            // Should fail
            subject.OnNext(document);

            A.CallTo(() => elasticClient.Index(index, "documentType", "documentBody")).MustNotHaveHappened();

            A.CallTo(() => document.IndexName).Returns("tempIndex");
            A.CallTo(() => document.Type).Returns("documentType");
            A.CallTo(() => document.Body).Returns("documentBody");

            // Should complete
            subject.OnNext(document);

            A.CallTo(() => elasticClient.Index(index, "documentType", "documentBody")).MustHaveHappened(Repeated.Exactly.Once);
        }
    }
}

{'fraction_non_alphanumeric': 0.1323249299719888, 'fraction_numerical': 0.001568627450980392, 'mean_word_length': 2.6255077173030057, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3869022', 'n_tokens_mistral': 2480, 'n_tokens_neox': 2349, 'n_words': 398}