<commit_before>/*******************************************************************************
 * Copyright (c) 2014
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    <NAME> - initial API and implementation and/or initial documentation
 *    <NAME>
 *******************************************************************************/
package org.splevo.vpm.analyzer.semantic;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import org.apache.log4j.Logger;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.graphstream.graph.Node;
import org.splevo.vpm.analyzer.AbstractVPMAnalyzer;
import org.splevo.vpm.analyzer.VPMAnalyzerResult;
import org.splevo.vpm.analyzer.VPMEdgeDescriptor;
import org.splevo.vpm.analyzer.config.BooleanConfiguration;
import org.splevo.vpm.analyzer.config.ChoiceConfiguration;
import org.splevo.vpm.analyzer.config.NumericConfiguration;
import org.splevo.vpm.analyzer.config.StringConfiguration;
import org.splevo.vpm.analyzer.config.VPMAnalyzerConfigurationSet;
import org.splevo.vpm.analyzer.graph.VPMGraph;
import org.splevo.vpm.analyzer.semantic.extensionpoint.SemanticContent;
import org.splevo.vpm.analyzer.semantic.extensionpoint.SemanticContentProvider;
import org.splevo.vpm.analyzer.semantic.extensionpoint.SemanticContentProviderRegistry;
import org.splevo.vpm.analyzer.semantic.extensionpoint.UnsupportedSoftwareElementException;
import org.splevo.vpm.analyzer.semantic.lucene.Indexer;
import org.splevo.vpm.analyzer.semantic.lucene.Stemming;
import org.splevo.vpm.analyzer.semantic.lucene.finder.SharedTermFinder;
import org.splevo.vpm.software.SoftwareElement;
import org.splevo.vpm.variability.Variant;
import org.splevo.vpm.variability.VariationPoint;

import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Table;
import com.google.common.collect.Table.Cell;

/**
 * <h2>What it does</h2>
 * <p>
 * The semantic relationship VPMAnalazer analyzer is able to find semantic relationships between
 * several {@link VariationPoint}s. Several configurations allow a customized search, just as
 * needed.
 * </p>
 *
 * <h2>How it works</h2>
 * <p>
 * As a first step, the analyzer extracts all relevant content from a VPMGraph and stores that
 * within a Lucene index. Through storing additional informations about the indexed text, Lucene
 * provides the ability to extract vectors from given index content. The Analyzer uses several
 * Finders to search for semantic dependencies. Those results can be displayed within the VPMGraph
 * or the Refinement Browser.
 * </p>
 */
public class SemanticVPMAnalyzer extends AbstractVPMAnalyzer {

    /** The relationship label of the analyzer. */
    private static final String RELATIONSHIP_LABEL_SEMANTIC = "Semantic";

    /** The displayed name of the analyzer. */
    private static final String DISPLAYED_NAME = "Semantic VPM Analyzer";

    /** The logger for this class. */
    private Logger logger = Logger.getLogger(SemanticVPMAnalyzer.class);

    /** The indexer instance. */
    private Indexer indexer = Indexer.getInstance();

    /** The configuration-object for the include comments configuration. */
    private BooleanConfiguration includeCommentsConfig = new BooleanConfiguration(Config.CONFIG_ID_INCLUDE_COMMENTS,
            Config.LABEL_INCLUDE_COMMENTS, null, Config.DEFAULT_INCLUDE_COMMENTS);

    /** The configuration-object for the split on case change configuration. */
    private BooleanConfiguration splitCamelCaseConfig = new BooleanConfiguration(Config.CONFIG_ID_SPLIT_CAMEL_CASE,
            Config.LABEL_SPLIT_CAMEL_CASE, null, Config.DEFAULT_SPLIT_CAMEL_CASE);

    /** The configuration-object for the stemming to be used. */
    private ChoiceConfiguration stemmingConfig = new ChoiceConfiguration(Config.CONFIG_ID_STEMMING,
            Config.LABEL_STEMMING, null, Config.DEFAULT_STEMMING, Config.AVAILABLEVALUES_STEMMING);

    /** The configuration-object for the stop words configuration. */
    private StringConfiguration stopWordsConfig = new StringConfiguration(Config.CONFIG_ID_STOP_WORDS,
            Config.LABEL_STOP_WORDS, Config.EXPL_STOP_WORDS, Config.DEFAULT_STOP_WORDS_HOST);

    /** The configuration-object for the minimum number of shared terms configuration. */
    private NumericConfiguration minSharedTermConfig = new NumericConfiguration(Config.CONFIG_ID_SHARED_TERM_MINIMUM,
            Config.LABEL_SHARED_TERM_MINIMUM, Config.EXPL_SHARED_TERM_MINIMUM, Config.DEFAULT_SHARED_TERM_MINIMUM, 1,
            1, -1, 0);

    /** The configuration-object for the log indexed terms configuration. */
    private BooleanConfiguration logIndexedTermsConfig = new BooleanConfiguration(Config.CONFIG_ID_LOG_INDEXED_TERMS,
            Config.LABEL_LOG_INDEXED_TERMS, null, Config.DEFAULT_LOG_INDEXED_TERMS);

    @Override
    public VPMAnalyzerResult analyze(VPMGraph vpmGraph) {
        if (vpmGraph == null) {
            throw new IllegalArgumentException();
        }
        if (vpmGraph.getNodeCount() == 0) {
            logger.info("Got empty VPM Graph. No analysis executed.");
            return null;
        }

        // Fill the index.
        try {
            fillIndex(vpmGraph);
        } catch (Exception e) {
            logger.error("Cannot write Index. Close all open IndexWriters first.", e);<commit_msg>bugfix: added missing parameter<commit_after>/*******************************************************************************
 * Copyright (c) 2014
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    <NAME> - initial API and implementation and/or initial documentation
 *    <NAME>
 *******************************************************************************/
package org.splevo.vpm.analyzer.semantic;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import org.apache.log4j.Logger;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.graphstream.graph.Node;
import org.splevo.vpm.analyzer.AbstractVPMAnalyzer;
import org.splevo.vpm.analyzer.VPMAnalyzerResult;
import org.splevo.vpm.analyzer.VPMEdgeDescriptor;
import org.splevo.vpm.analyzer.config.BooleanConfiguration;
import org.splevo.vpm.analyzer.config.ChoiceConfiguration;
import org.splevo.vpm.analyzer.config.NumericConfiguration;
import org.splevo.vpm.analyzer.config.StringConfiguration;
import org.splevo.vpm.analyzer.config.VPMAnalyzerConfigurationSet;
import org.splevo.vpm.analyzer.graph.VPMGraph;
import org.splevo.vpm.analyzer.semantic.extensionpoint.SemanticContent;
import org.splevo.vpm.analyzer.semantic.extensionpoint.SemanticContentProvider;
import org.splevo.vpm.analyzer.semantic.extensionpoint.SemanticContentProviderRegistry;
import org.splevo.vpm.analyzer.semantic.extensionpoint.UnsupportedSoftwareElementException;
import org.splevo.vpm.analyzer.semantic.lucene.Indexer;
import org.splevo.vpm.analyzer.semantic.lucene.Stemming;
import org.splevo.vpm.analyzer.semantic.lucene.finder.SharedTermFinder;
import org.splevo.vpm.software.SoftwareElement;
import org.splevo.vpm.variability.Variant;
import org.splevo.vpm.variability.VariationPoint;

import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Table;
import com.google.common.collect.Table.Cell;

/**
 * <h2>What it does</h2>
 * <p>
 * The semantic relationship VPMAnalazer analyzer is able to find semantic relationships between
 * several {@link VariationPoint}s. Several configurations allow a customized search, just as
 * needed.
 * </p>
 *
 * <h2>How it works</h2>
 * <p>
 * As a first step, the analyzer extracts all relevant content from a VPMGraph and stores that
 * within a Lucene index. Through storing additional informations about the indexed text, Lucene
 * provides the ability to extract vectors from given index content. The Analyzer uses several
 * Finders to search for semantic dependencies. Those results can be displayed within the VPMGraph
 * or the Refinement Browser.
 * </p>
 */
public class SemanticVPMAnalyzer extends AbstractVPMAnalyzer {

    /** The relationship label of the analyzer. */
    private static final String RELATIONSHIP_LABEL_SEMANTIC = "Semantic";

    /** The displayed name of the analyzer. */
    private static final String DISPLAYED_NAME = "Semantic VPM Analyzer";

    /** The logger for this class. */
    private Logger logger = Logger.getLogger(SemanticVPMAnalyzer.class);

    /** The indexer instance. */
    private Indexer indexer = Indexer.getInstance();

    /** The configuration-object for the include comments configuration. */
    private BooleanConfiguration includeCommentsConfig = new BooleanConfiguration(Config.CONFIG_ID_INCLUDE_COMMENTS,
            Config.LABEL_INCLUDE_COMMENTS, null, Config.DEFAULT_INCLUDE_COMMENTS);

    /** The configuration-object for the split on case change configuration. */
    private BooleanConfiguration splitCamelCaseConfig = new BooleanConfiguration(Config.CONFIG_ID_SPLIT_CAMEL_CASE,
            Config.LABEL_SPLIT_CAMEL_CASE, null, Config.DEFAULT_SPLIT_CAMEL_CASE);

    /** The configuration-object for the stemming to be used. */
    private ChoiceConfiguration stemmingConfig = new ChoiceConfiguration(Config.CONFIG_ID_STEMMING,
            Config.LABEL_STEMMING, null, Config.DEFAULT_STEMMING, Config.AVAILABLEVALUES_STEMMING);

    /** The configuration-object for the stop words configuration. */
    private StringConfiguration stopWordsConfig = new StringConfiguration(Config.CONFIG_ID_STOP_WORDS,
            Config.LABEL_STOP_WORDS, Config.EXPL_STOP_WORDS, Config.DEFAULT_STOP_WORDS_HOST, false);

    /** The configuration-object for the minimum number of shared terms configuration. */
    private NumericConfiguration minSharedTermConfig = new NumericConfiguration(Config.CONFIG_ID_SHARED_TERM_MINIMUM,
            Config.LABEL_SHARED_TERM_MINIMUM, Config.EXPL_SHARED_TERM_MINIMUM, Config.DEFAULT_SHARED_TERM_MINIMUM, 1,
            1, -1, 0);

    /** The configuration-object for the log indexed terms configuration. */
    private BooleanConfiguration logIndexedTermsConfig = new BooleanConfiguration(Config.CONFIG_ID_LOG_INDEXED_TERMS,
            Config.LABEL_LOG_INDEXED_TERMS, null, Config.DEFAULT_LOG_INDEXED_TERMS);

    @Override
    public VPMAnalyzerResult analyze(VPMGraph vpmGraph) {
        if (vpmGraph == null) {
            throw new IllegalArgumentException();
        }
        if (vpmGraph.getNodeCount() == 0) {
            logger.info("Got empty VPM Graph. No analysis executed.");
            return null;
        }

        // Fill the index.
        try {
            fillIndex(vpmGraph);
        } catch (Exception e) {
            logger.error("Cannot write Index. Close all open IndexWriters first.", e);
{'fraction_non_alphanumeric': 0.11041145518847581, 'fraction_numerical': 0.003105322177175882, 'mean_word_length': 4.486985328916233, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '836447', 'n_tokens_mistral': 3420, 'n_tokens_neox': 3257, 'n_words': 799}