<commit_before>/*
 * LingoTest.java Created on 2004-06-23
 */
package com.stachoodev.carrot.filter.lingo.algorithm;

import java.util.*;

import com.dawidweiss.carrot.core.local.clustering.*;
import com.dawidweiss.carrot.util.tokenizer.*;

import junit.framework.*;

/**
 * @author stachoo
 */
public class LingoTest extends TestCase
{
    /** A helper SnippetTokenizer */
    private SnippetTokenizer snippetTokenizer;

    /*
     * (non-Javadoc)
     * 
     * @see junit.framework.TestCase#setUp()
     */
    protected void setUp() throws Exception
    {
        super.setUp();
        snippetTokenizer = new SnippetTokenizer();
    }

    /**
     * 
     */
    public void testNoClusters()
    {
        TokenizedDocument document01 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "a b c d e"));
        TokenizedDocument document02 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "f g h i j"));
        TokenizedDocument document03 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "k l m n o"));

        List documentList = Arrays.asList(new TokenizedDocument []
        { document01, document02, document03 });

        Lingo lingo = new Lingo();
        lingo.setCandidateLabelsCount(10);
        List clusters = lingo.cluster(documentList);
        
        assertEquals("No clusters generated", 0, clusters.size());
    }

    /**
     * 
     */
    public void testLabelDiscovery()
    {
        TokenizedDocument document01 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "large scale singular value computations", "en"));
        TokenizedDocument document02 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "software library for the sparse singular value decomposition",
                "en"));
        TokenizedDocument document03 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "introduction to modern information retrieval", "en"));
        TokenizedDocument document04 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "using linear algebra for intelligent information retrieval",
                "en"));
        TokenizedDocument document05 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "matrix computations", "en"));
        TokenizedDocument document06 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "singular value analysis of cryptograms", "en"));
        TokenizedDocument document07 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "automatic information organization", "en"));

        List documentList = Arrays.asList(new TokenizedDocument []
        { document01, document02, document03, document04, document05,
         document06, document07 });

        Lingo lingo = new Lingo();
        lingo.setCandidateLabelsCount(10);
        List clusters = lingo.cluster(documentList);

        for (Iterator iter = clusters.iterator(); iter.hasNext();)
        {
            LingoRawCluster cluster = (LingoRawCluster) iter.next();
            System.out.println(cluster.getFullInfo() + "\n");
        }
    }
}<commit_msg>Added: more tests<commit_after>/*
 * Carrot2 Project
 * Copyright (C) 2002-2004, <NAME>
 * Portions (C) Contributors listed in carrot2.CONTRIBUTORS file.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the CVS checkout or at:
 * http://www.cs.put.poznan.pl/dweiss/carrot2.LICENSE
 */
package com.stachoodev.carrot.filter.lingo.algorithm;

import java.util.*;

import com.dawidweiss.carrot.core.local.clustering.*;
import com.dawidweiss.carrot.util.tokenizer.*;
import com.stachoodev.carrot.filter.lingo.model.*;

import junit.framework.*;

/**
 * Simple unit tests for the Lingo algorithm.
 * 
 * @author <NAME>
 * @version $Revision$
 */
public class LingoTest extends TestCase
{
    /** A helper SnippetTokenizer */
    private SnippetTokenizer snippetTokenizer;

    /*
     * (non-Javadoc)
     * 
     * @see junit.framework.TestCase#setUp()
     */
    protected void setUp() throws Exception
    {
        super.setUp();
        snippetTokenizer = new SnippetTokenizer();
    }

    /**
     *  
     */
    public void testNoClusters()
    {
        TokenizedDocument document01 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "a b c d e"));
        TokenizedDocument document02 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "f g h i j"));
        TokenizedDocument document03 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "k l m n o"));

        List documentList = Arrays.asList(new TokenizedDocument []
        { document01, document02, document03 });

        Lingo lingo = new Lingo();
        lingo.setCandidateLabelsCount(10);
        List clusters = lingo.cluster(documentList);

        assertEquals("No clusters generated", 0, clusters.size());
    }

    /**
     *  
     */
    public void testIdfOmittedDocuments()
    {
        TokenizedDocument document00 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "value", "en"));
        TokenizedDocument document01 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "large scale singular value computations", "en"));
        TokenizedDocument document02 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "software library for the sparse singular value decomposition",
                "en"));
        TokenizedDocument document03 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "value introduction to modern information retrieval", "en"));
        TokenizedDocument document04 = snippetTokenizer
            .tokenize(new RawDocumentSnippet(
                "",
                "using linear algebra for intelligent information value retrieval",
                "en"));
        TokenizedDocument document05 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "value of matrix computations", "en"));
        TokenizedDocument document06 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "singular value analysis of cryptograms", "en"));
        TokenizedDocument document07 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "automatic information value organization", "en"));

        List documentList = Arrays.asList(new TokenizedDocument []
        { document00, document01, document02, document03, document04,
         document05, document06, document07 });

        Map parameters = new HashMap();
        parameters.put(Lingo.PARAMETER_TD_MATRIX_BUILDING_STRATEGY,
            new TfIdfTdMatrixBuildingStrategy());

        Lingo lingo = new Lingo(parameters);
        lingo.setCandidateLabelsCount(2);
        List clusters = lingo.cluster(documentList);

        // Just check if the first document ended up in the Other Topics group
        assertTrue("document00 in the Other Topics group",
            ((RawCluster) clusters.get(clusters.size() - 1)).getDocuments()
                .contains(
                    document00
                        .getProperty(TokenizedDocument.PROPERTY_RAW_DOCUMENT)));
    }

    /**
     *  
     */
    public void testTfOmittedDocuments()
    {
        TokenizedDocument document00 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "library cryptograms", "en"));
        TokenizedDocument document08 = snippetTokenizer
        .tokenize(new RawDocumentSnippet("", "analysis decomposition", "en"));
        TokenizedDocument document01 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "large scale singular value computations", "en"));
        TokenizedDocument document02 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "software library for the sparse singular value decomposition",
                "en"));
        TokenizedDocument document03 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "introduction to modern information retrieval", "en"));
        TokenizedDocument document04 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "using linear algebra for intelligent information retrieval",
                "en"));
        TokenizedDocument document05 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "matrix computations", "en"));
        TokenizedDocument document06 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "singular value analysis of cryptograms", "en"));
        TokenizedDocument document07 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "automatic information organization", "en"));

        List documentList = Arrays.asList(new TokenizedDocument []
        { document01, document00, document02, document03, document04,
         document05, document08, document06, document07 });

        Map parameters = new HashMap();
        parameters.put(Lingo.PARAMETER_TD_MATRIX_BUILDING_STRATEGY,
            new TfIdfTdMatrixBuildingStrategy());

        Lingo lingo = new Lingo(parameters);
        lingo.setCandidateLabelsCount(2);
        List clusters = lingo.cluster(documentList);

        // Documents 00 08 05 in the Other Topics group
        assertTrue("document00 in the Other Topics group",
            ((RawCluster) clusters.get(clusters.size() - 1)).getDocuments()
                .contains(
                    document00
                        .getProperty(TokenizedDocument.PROPERTY_RAW_DOCUMENT)));
        assertTrue("document08 in the Other Topics group",
            ((RawCluster) clusters.get(clusters.size() - 1)).getDocuments()
                .contains(
                    document08
                        .getProperty(TokenizedDocument.PROPERTY_RAW_DOCUMENT)));
        assertTrue("document05 in the Other Topics group",
            ((RawCluster) clusters.get(clusters.size() - 1)).getDocuments()
                .contains(
                    document05
                        .getProperty(TokenizedDocument.PROPERTY_RAW_DOCUMENT)));
    }

    /**
     *  
     */
    public void testLabelDiscovery()
    {
        TokenizedDocument document01 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "large scale singular value computations", "en"));
        TokenizedDocument document02 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "software library for the sparse singular value decomposition",
                "en"));
        TokenizedDocument document03 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "introduction to modern information retrieval", "en"));
        TokenizedDocument document04 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "using linear algebra for intelligent information retrieval",
                "en"));
        TokenizedDocument document05 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("", "matrix computations", "en"));
        TokenizedDocument document06 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "singular value analysis of cryptograms", "en"));
        TokenizedDocument document07 = snippetTokenizer
            .tokenize(new RawDocumentSnippet("",
                "automatic information organization", "en"));

        List documentList = Arrays.asList(new TokenizedDocument []
        { document01, document02, document03, document04, document05,
         document06, document07 });

        Lingo lingo = new Lingo();
        lingo.setCandidateLabelsCount(10);
        List clusters = lingo.cluster(documentList);

        // No more than three clusters generated
        assertTrue("No more than three clusters", clusters.size() <= 3);
    }
}
{'fraction_non_alphanumeric': 0.09713718351812013, 'fraction_numerical': 0.01712725467483038, 'mean_word_length': 2.4455530216647663, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5665959', 'n_tokens_mistral': 3250, 'n_tokens_neox': 2968, 'n_words': 764}