<commit_before>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;

namespace NuGetGallery
{
    public class PerFieldAnalyzer : PerFieldAnalyzerWrapper
    {
        public PerFieldAnalyzer()
            : base(new StandardAnalyzer(LuceneCommon.LuceneVersion), CreateFieldAnalyzers())
        {
        }

        private static IDictionary CreateFieldAnalyzers()
        {
            return new Dictionary<string, Analyzer>(StringComparer.OrdinalIgnoreCase)
            {
                { "Title", new TitleAnalyzer() }
            };
        }

        class TitleAnalyzer : Analyzer
        {
            private readonly Analyzer innerAnalyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);

            public override TokenStream TokenStream(string fieldName, TextReader reader)
            {
                // Split the title based on IdSeparators, then run it through the standardAnalyzer
                string title = reader.ReadToEnd();
                string partiallyTokenized = String.Join(" ", title.Split(LuceneIndexingService.IdSeparators, StringSplitOptions.RemoveEmptyEntries));
                return innerAnalyzer.TokenStream(fieldName, new StringReader(partiallyTokenized));
            }
        }
    }
}<commit_msg>Stop filtering out stop words in package ids and titles.<commit_after>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;

namespace NuGetGallery
{
    public class PerFieldAnalyzer : PerFieldAnalyzerWrapper
    {
        public PerFieldAnalyzer()
            : base(new StandardAnalyzer(LuceneCommon.LuceneVersion), CreateFieldAnalyzers())
        {
        }

        private static IDictionary CreateFieldAnalyzers()
        {
            // For idAnalyzer we use the 'standard analyzer' but with no stop words (In, Of, The, etc are indexed).
            var stopWords = new Hashtable();
            StandardAnalyzer idAnalyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion, stopWords);

            return new Dictionary<string, Analyzer>(StringComparer.OrdinalIgnoreCase)
            {
                { "Id", idAnalyzer },
                { "Title", new TitleAnalyzer() },
            };
        }

        class TitleAnalyzer : Analyzer
        {
            private readonly StandardAnalyzer innerAnalyzer;

            public TitleAnalyzer()
            {
                // For innerAnalyzer we use the 'standard analyzer' but with no stop words (In, Of, The, etc are indexed).
                var stopWords = new Hashtable();
                innerAnalyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion, stopWords);
            }

            public override TokenStream TokenStream(string fieldName, TextReader reader)
            {
                // Split the title based on IdSeparators, then run it through the innerAnalyzer
                string title = reader.ReadToEnd();
                string partiallyTokenized = String.Join(" ", title.Split(LuceneIndexingService.IdSeparators, StringSplitOptions.RemoveEmptyEntries));
                return innerAnalyzer.TokenStream(fieldName, new StringReader(partiallyTokenized));
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.07378524295140972, 'fraction_numerical': 0.0, 'mean_word_length': 2.3961303462321792, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4816740', 'n_tokens_mistral': 842, 'n_tokens_neox': 798, 'n_words': 212}