<commit_before>﻿using System;
using DocumentTable;

namespace Resin.Analysis
{
    public class TfIdf : IScoringScheme
    {
        private readonly double _idf;

        public double Idf
        {
            get { return _idf; }
        }

        /// <summary>
        /// Create scorer. 
        /// On tf-idf: 
        /// https://lucene.apache.org/core/6_4_1/core/org/apache/lucene/search/similarities/TFIDFSimilarity.html 
        /// https://en.wikipedia.org/wiki/Tf%E2%80%93idf
        /// </summary>
        /// <param name="docsInCorpus"></param>
        /// <param name="docsWithTerm"></param>
        public TfIdf(int docsInCorpus, int docsWithTerm)
        {
            _idf = Math.Log10(docsInCorpus / (double)docsWithTerm);
        }

        public double Score(DocumentPosting posting)
        {
            return Math.Sqrt(posting.Count) * _idf;
        }
    }
}<commit_msg>probabilistic inverse document frequency & log-normalized term frequency<commit_after>﻿using System;
using DocumentTable;

namespace Resin.Analysis
{
    public class TfIdf : IScoringScheme
    {
        private readonly double _idf;

        public double Idf
        {
            get { return _idf; }
        }

        /// <summary>
        /// Create scorer. 
        /// On tf-idf: 
        /// https://en.wikipedia.org/wiki/Tf%E2%80%93idf
        /// </summary>
        /// <param name="docsInCorpus"></param>
        /// <param name="docsWithTerm"></param>
        public TfIdf(int docsInCorpus, int docsWithTerm)
        {
            // probabilistic inverse document frequency
            _idf = Math.Log10(docsInCorpus - docsWithTerm / (double)docsWithTerm);
        }

        public double Score(DocumentPosting posting)
        {
            // log-normalized term frequency
            return 1 + Math.Log10(posting.Count) * _idf;
        }
    }
}
{'fraction_non_alphanumeric': 0.11761546723952739, 'fraction_numerical': 0.010741138560687433, 'mean_word_length': 1.7805970149253731, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2340079', 'n_tokens_mistral': 584, 'n_tokens_neox': 554, 'n_words': 108}