<commit_before>
/*
 * Carrot2 project.
 *
 * Copyright (C) 2002-2010, <NAME>, <NAME>.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the repository checkout or at:
 * http://www.carrot2.org/carrot2.LICENSE
 */

package org.carrot2.clustering.lingo;

import static org.fest.assertions.Assertions.assertThat;

import org.carrot2.text.vsm.TfTermWeighting;
import org.junit.Before;
import org.junit.Test;

/**
 * Test cases for label building in {@link ClusterBuilder}.
 */
public class ClusterLabelBuilderTest extends LingoProcessingComponentTestBase
{
    /** Matrix reducer needed for test */
    private TermDocumentMatrixReducer reducer;

    /** Label builder under tests */
    private ClusterBuilder clusterBuilder;

    @Before
    public void setUpClusterLabelBuilder()
    {
        clusterBuilder = new ClusterBuilder();
        clusterBuilder.labelAssigner = new SimpleLabelAssigner();
        reducer = new TermDocumentMatrixReducer();
    }

    @Test
    public void testEmpty()
    {
        check(new int [0]);
    }

    @Test
    public void testNoPhrases()
    {
        createDocuments("", "aa . bb", "", "bb . cc", "", "cc . aa");
        final int [] expectedFeatureIndex = new int []
        {
            0, 1, 2
        };

        reducer.desiredClusterCountBase = 30;
        check(expectedFeatureIndex);
    }

    @Test
    public void testSinglePhraseNoSingleWords()
    {
        createDocuments("aa bb", "aa bb", "aa bb", "aa bb");

        final int [] expectedFeatureIndex = new int []
        {
            2
        };

        reducer.desiredClusterCountBase = 10;
        check(expectedFeatureIndex);
    }

    @Test
    public void testSinglePhraseSingleWords()
    {
        createDocuments("aa bb", "aa bb", "cc", "cc", "aa bb", "aa bb . cc");
        clusterBuilder.phraseLabelBoost = 0.5;

        final int [] expectedFeatureIndex = new int []
        {
            2, 3
        };

        reducer.desiredClusterCountBase = 15;
        check(expectedFeatureIndex);
    }

    @Test
    public void testQueryWordsRemoval()
    {
        createDocuments("query word . aa", "query word . aa", "query . word",
            "query . word . aa");
        clusterBuilder.phraseLabelBoost = 0.5;

        final int [] expectedFeatureIndex = new int []
        {
            0
        };

        reducer.desiredClusterCountBase = 10;
        createPreprocessingContext("query word");
        check(expectedFeatureIndex);
    }

    @Test
    public void testExternalFeatureScores()
    {
        createDocuments("aa bb", "aa bb", "cc", "cc", "cc", "cc", "aa bb", "aa bb", "dd",
            "dd", "dd", "dd", "ee ff", "ee ff", "ee ff", "ee ff");
        clusterBuilder.phraseLabelBoost = 0.5;
        reducer.desiredClusterCountBase = 15;

        final int [] expectedFeatureIndex = new int []
        {
            6, 7, 2, 3
        };
        check(expectedFeatureIndex);

        // Make a copy of feature indices
        final int [] featureIndex = lingoContext.preprocessingContext.allLabels.featureIndex;

        for (int i = 0; i < featureIndex.length; i++)
        {
            clusterBuilder.featureScorer = new OneLabelFeatureScorer(i, 2);
            check(new int []
            {
                featureIndex[i], featureIndex[i], featureIndex[i], featureIndex[i]
            });
        }
    }

    private static class OneLabelFeatureScorer implements IFeatureScorer
    {
        private int labelIndex;
        private double score;

        OneLabelFeatureScorer(int labelIndex, double score)
        {
            this.labelIndex = labelIndex;
            this.score = score;
        }

        public double [] getFeatureScores(LingoProcessingContext lingoContext)
        {
            final double [] scores = new double [lingoContext.preprocessingContext.allLabels.featureIndex.length];
            scores[labelIndex] = score;
            return scores;
        }
    }

    private void check(int [] expectedFeatureIndex)
    {
        buildTermDocumentMatrix();
        reducer.reduce(lingoContext);
        clusterBuilder.buildLabels(lingoContext, new TfTermWeighting());

        assertThat(lingoContext.clusterLabelFeatureIndex).as("clusterLabelFeatureIndex")
            .containsOnly(expectedFeatureIndex);
    }
}<commit_msg>carrot2-algorithm-lingo: fixing a failing test<commit_after>
/*
 * Carrot2 project.
 *
 * Copyright (C) 2002-2010, <NAME>, <NAME>.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the repository checkout or at:
 * http://www.carrot2.org/carrot2.LICENSE
 */

package org.carrot2.clustering.lingo;

import static org.fest.assertions.Assertions.assertThat;

import org.carrot2.text.vsm.TfTermWeighting;
import org.junit.Before;
import org.junit.Test;

/**
 * Test cases for label building in {@link ClusterBuilder}.
 */
public class ClusterLabelBuilderTest extends LingoProcessingComponentTestBase
{
    /** Matrix reducer needed for test */
    private TermDocumentMatrixReducer reducer;

    /** Label builder under tests */
    private ClusterBuilder clusterBuilder;

    @Before
    public void setUpClusterLabelBuilder()
    {
        clusterBuilder = new ClusterBuilder();
        clusterBuilder.labelAssigner = new SimpleLabelAssigner();
        reducer = new TermDocumentMatrixReducer();
    }

    @Test
    public void testEmpty()
    {
        buildModelAndCheck(new int [0]);
    }

    @Test
    public void testNoPhrases()
    {
        createDocuments("", "aa . bb", "", "bb . cc", "", "cc . aa");
        final int [] expectedFeatureIndex = new int []
        {
            0, 1, 2
        };

        reducer.desiredClusterCountBase = 30;
        buildModelAndCheck(expectedFeatureIndex);
    }

    @Test
    public void testSinglePhraseNoSingleWords()
    {
        createDocuments("aa bb", "aa bb", "aa bb", "aa bb");

        final int [] expectedFeatureIndex = new int []
        {
            2
        };

        reducer.desiredClusterCountBase = 10;
        buildModelAndCheck(expectedFeatureIndex);
    }

    @Test
    public void testSinglePhraseSingleWords()
    {
        createDocuments("aa bb", "aa bb", "cc", "cc", "aa bb", "aa bb . cc");
        clusterBuilder.phraseLabelBoost = 0.5;

        final int [] expectedFeatureIndex = new int []
        {
            2, 3
        };

        reducer.desiredClusterCountBase = 15;
        buildModelAndCheck(expectedFeatureIndex);
    }

    @Test
    public void testQueryWordsRemoval()
    {
        createDocuments("query word . aa", "query word . aa", "query . word",
            "query . word . aa");
        clusterBuilder.phraseLabelBoost = 0.5;

        final int [] expectedFeatureIndex = new int []
        {
            0
        };

        reducer.desiredClusterCountBase = 10;
        createPreprocessingContext("query word");
        buildModelAndCheck(expectedFeatureIndex);
    }

    @Test
    public void testExternalFeatureScores()
    {
        createDocuments("aa bb", "aa bb", "cc", "cc", "cc", "cc", "aa bb", "aa bb", "dd",
            "dd", "dd", "dd", "ee ff", "ee ff", "ee ff", "ee ff");
        clusterBuilder.phraseLabelBoost = 0.5;
        reducer.desiredClusterCountBase = 15;

        final int [] expectedFeatureIndex = new int []
        {
            6, 7, 2, 3
        };
        buildModelAndCheck(expectedFeatureIndex);

        // Make a copy of feature indices
        final int [] featureIndex = lingoContext.preprocessingContext.allLabels.featureIndex;

        for (int i = 0; i < featureIndex.length; i++)
        {
            clusterBuilder.featureScorer = new OneLabelFeatureScorer(i, 2);
            check(new int []
            {
                featureIndex[i], featureIndex[i], featureIndex[i], featureIndex[i]
            });
        }
    }

    private static class OneLabelFeatureScorer implements IFeatureScorer
    {
        private int labelIndex;
        private double score;

        OneLabelFeatureScorer(int labelIndex, double score)
        {
            this.labelIndex = labelIndex;
            this.score = score;
        }

        public double [] getFeatureScores(LingoProcessingContext lingoContext)
        {
            final double [] scores = new double [lingoContext.preprocessingContext.allLabels.featureIndex.length];
            scores[labelIndex] = score;
            return scores;
        }
    }

    private void buildModelAndCheck(int [] expectedFeatureIndex)
    {
        buildTermDocumentMatrix();
        reducer.reduce(lingoContext);
        check(expectedFeatureIndex);
    }

    private void check(int [] expectedFeatureIndex)
    {
        clusterBuilder.buildLabels(lingoContext, new TfTermWeighting());
        assertThat(lingoContext.clusterLabelFeatureIndex).as("clusterLabelFeatureIndex")
            .containsOnly(expectedFeatureIndex);
    }
}
{'fraction_non_alphanumeric': 0.10597065083454688, 'fraction_numerical': 0.009969754676823121, 'mean_word_length': 2.3152617898254735, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '684892', 'n_tokens_mistral': 2620, 'n_tokens_neox': 2434, 'n_words': 609}