<commit_before>package se.imagick.ta.language;

import se.imagick.ta.filter.TextUtils;
import se.imagick.ta.tfidf.Term;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.IntStream;

/**
 * Detects a document language by filtering it with several stop word lists.
 * The language associated with the stop word list that filters out the most words
 * is assumed to be the correct one. A rather naive method that needs more than a few words.
 * Could also be used to detect the document category by using stop word lists that
 * contains category specific words.
 */
public class LanguageDetector {

    public static String detect(String document, StopWordList... stopWordsListCollection) {

        IntStream chars = document.codePoints();
        StringBuilder documentSb = new StringBuilder();
        chars.filter(TextUtils::isAlphaOrSpace).forEach(e -> documentSb.append(Character.toChars(e)));

        List<Term> termList = TextUtils.getAllTerms(documentSb.toString(), 1, new ArrayList<>(), null);
        String reccordLanguage = "Not detected";
        double reccord = 0;

        for(StopWordList stopWords : stopWordsListCollection) {

            double noOfStopWordsInText = (double) termList.stream().filter(term -> stopWords.isStopWord(term.getJoinedTerm())).count();

            if (noOfStopWordsInText > reccord) {
                reccord = noOfStopWordsInText;
                reccordLanguage = stopWords.getLanguage();
            }
        }

        return reccordLanguage;
    }
}
<commit_msg>Added todo.<commit_after>package se.imagick.ta.language;

import se.imagick.ta.filter.TextUtils;
import se.imagick.ta.tfidf.Term;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.IntStream;

/**
 * Detects a document language by filtering it with several stop word lists.
 * The language associated with the stop word list that filters out the most words
 * is assumed to be the correct one. A rather naive method that needs more than a few words.
 * Could also be used to detect the document category by using stop word lists that
 * contains category specific words.
 * TODO add functionality for retrieving percentage of words sorted out by the stop word list.
 */
public class LanguageDetector {

    public static String detect(String document, StopWordList... stopWordsListCollection) {

        IntStream chars = document.codePoints();
        StringBuilder documentSb = new StringBuilder();
        chars.filter(TextUtils::isAlphaOrSpace).forEach(e -> documentSb.append(Character.toChars(e)));

        List<Term> termList = TextUtils.getAllTerms(documentSb.toString(), 1, new ArrayList<>(), null);
        String reccordLanguage = "Not detected";
        double reccord = 0;

        for(StopWordList stopWords : stopWordsListCollection) {

            double noOfStopWordsInText = (double) termList.stream().filter(term -> stopWords.isStopWord(term.getJoinedTerm())).count();

            if (noOfStopWordsInText > reccord) {
                reccord = noOfStopWordsInText;
                reccordLanguage = stopWords.getLanguage();
            }
        }

        return reccordLanguage;
    }
}

{'fraction_non_alphanumeric': 0.08850393700787401, 'fraction_numerical': 0.0012598425196850393, 'mean_word_length': 3.309362279511533, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '825628', 'n_tokens_mistral': 883, 'n_tokens_neox': 829, 'n_words': 266}