<commit_before>﻿using Lezen.Core.Search;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Index;
using Lucene.Net.Store;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using FluentAssertions;
using Xunit;

namespace Lezen.Core.Test.Search
{
    public class _01_QueryIndex
    {
        [Fact]
        public void Test()
        {
            using (var temp = new TempDirectory(System.IO.Directory.GetCurrentDirectory()))
            using (var directory = FSDirectory.Open(temp.Path))
            {
                var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);

                IndexWriter.MaxFieldLength length = new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH);
                var writer = new IndexWriter(directory, analyzer, length);

                var documentFactory = new DocumentFactory();
                var searchItem1 = new SearchItem { EntityID = 1, Abstract = "abstract1", Keywords = new string[0], Text = "text1" };
                var searchItem2 = new SearchItem { EntityID = 2, Abstract = "abstract2", Keywords = new string[0], Text = "text2" };
                var searchItem3 = new SearchItem { EntityID = 3, Abstract = "abstract3", Keywords = new string[0], Text = "text3" };

                writer.AddDocument(documentFactory.Create(searchItem1));
                writer.AddDocument(documentFactory.Create(searchItem2));
                writer.AddDocument(documentFactory.Create(searchItem3));

                writer.Commit();

                writer.NumDocs().Should().Be(3);
                writer.Dispose();
            }<commit_msg>Lucene is too much for a unit test.<commit_after>﻿using Lezen.Core.Search;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Index;
using Lucene.Net.Store;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using FluentAssertions;
using Xunit;

namespace Lezen.Core.Test.Search
{
    public class _01_QueryIndex
    {
        [Fact(Skip="Too much for a unit test")]
        public void Test()
        {
            using (var temp = new TempDirectory(System.IO.Directory.GetCurrentDirectory()))
            using (var directory = FSDirectory.Open(temp.Path))
            {
                var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);

                IndexWriter.MaxFieldLength length = new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH);
                var writer = new IndexWriter(directory, analyzer, length);

                var documentFactory = new DocumentFactory();
                var searchItem1 = new SearchItem { EntityID = 1, Abstract = "abstract1", Keywords = new string[0], Text = "text1" };
                var searchItem2 = new SearchItem { EntityID = 2, Abstract = "abstract2", Keywords = new string[0], Text = "text2" };
                var searchItem3 = new SearchItem { EntityID = 3, Abstract = "abstract3", Keywords = new string[0], Text = "text3" };

                writer.AddDocument(documentFactory.Create(searchItem1));
                writer.AddDocument(documentFactory.Create(searchItem2));
                writer.AddDocument(documentFactory.Create(searchItem3));

                writer.Commit();

                writer.NumDocs().Should().Be(3);
                writer.Dispose();
            }
{'fraction_non_alphanumeric': 0.10341805433829973, 'fraction_numerical': 0.013438504236050247, 'mean_word_length': 2.596638655462185, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4730925', 'n_tokens_mistral': 948, 'n_tokens_neox': 898, 'n_words': 218}