<commit_before>// Copyright 2015 Georg-August-Universität Göttingen, Germany
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package de.ugoe.cs.cpdp.dataprocessing;

import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.stream.IntStream;

import org.apache.commons.collections4.list.SetUniqueList;
import org.apache.commons.math3.stat.correlation.SpearmansCorrelation;

import com.google.common.collect.Sets;

import de.ugoe.cs.cpdp.util.SortUtils;
import de.ugoe.cs.util.console.Console;
import weka.attributeSelection.AttributeSelection;
import weka.attributeSelection.CfsSubsetEval;
import weka.attributeSelection.GreedyStepwise;
import weka.classifiers.trees.J48;
import weka.core.Instances;

/**
 * <p>
 * Implements the OPTTOPk filter after P. He et al. (2015).
 * </p>
 * 
 * @author <NAME>
 */
public class TopMetricFilter implements ISetWiseProcessingStrategy {

    /**
     * Internally used correlation threshold.
     */
    double correlationThreshold = 0.5;
    
    /*
     * (non-Javadoc)
     * 
     * @see de.ugoe.cs.cpdp.IParameterizable#setParameter(java.lang.String)
     */
    @Override
    public void setParameter(String parameters) {
        if( parameters!=null && !parameters.equals("")) {
            correlationThreshold = Double.parseDouble(parameters);
        }
    }

    @Override
    public void apply(Instances testdata, SetUniqueList<Instances> traindataSet) {
        try {
            determineTopKAttributes(testdata, traindataSet);
        }
        catch (Exception e) {
            Console.printerr("Failure during metric selection: " + e.getMessage());
            throw new RuntimeException(e);
        }
    }

    private void determineTopKAttributes(Instances testdata, SetUniqueList<Instances> traindataSet) throws Exception {
        Integer[] counts = new Integer[traindataSet.get(0).numAttributes()-1];
        for( Instances traindata : traindataSet ) {
            J48 decisionTree = new J48();
            decisionTree.buildClassifier(traindata);
            int k=0;
            for( int j=0; j<traindata.numAttributes(); j++) {
                if(j!=traindata.classIndex()){
                    if( decisionTree.toString().contains(traindata.attribute(j).name()) ) {
                        if( counts[k]==null ){
                            counts[k] = 1;
                        } else {
                            counts[k] = counts[k]+1;
                        }
                    }
                    k++;
                }
            }
        }
        int[] topkIndex = new int[counts.length];
        IntStream.range(0,counts.length).forEach(val -> topkIndex[val] = val);
        SortUtils.quicksort(counts, topkIndex, true);
        
        // get CFSs for each training set
        List<Set<Integer>> cfsSets = new LinkedList<>();
        for( Instances traindata : traindataSet ) {
            AttributeSelection attsel = new AttributeSelection();
            CfsSubsetEval eval = new CfsSubsetEval();
            GreedyStepwise search = new GreedyStepwise();
            search.setSearchBackwards(true);
            attsel.setEvaluator(eval);
            attsel.setSearch(search);
            attsel.SelectAttributes(traindata);
            Set<Integer> cfsSet = new HashSet<>();
            for( int attr : attsel.selectedAttributes() ) {
                cfsSet.add(attr);
            }
            cfsSets.add(cfsSet);
        }
        
        double[] coverages = new double[topkIndex.length];
        for( Set<Integer> cfsSet : cfsSets ) {
            Set<Integer> topkSet = new HashSet<>();
            for( int k=0; k<topkIndex.length ; k++ ) {<commit_msg>* fixed bug in TopMetricFilter due to an initialization error<commit_after>// Copyright 2015 Georg-August-Universität Göttingen, Germany
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package de.ugoe.cs.cpdp.dataprocessing;

import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.stream.IntStream;

import org.apache.commons.collections4.list.SetUniqueList;
import org.apache.commons.math3.stat.correlation.SpearmansCorrelation;

import com.google.common.collect.Sets;

import de.ugoe.cs.cpdp.util.SortUtils;
import de.ugoe.cs.util.console.Console;
import weka.attributeSelection.AttributeSelection;
import weka.attributeSelection.CfsSubsetEval;
import weka.attributeSelection.GreedyStepwise;
import weka.classifiers.trees.J48;
import weka.core.Instances;

/**
 * <p>
 * Implements the OPTTOPk filter after P. He et al. (2015).
 * </p>
 * 
 * @author <NAME>
 */
public class TopMetricFilter implements ISetWiseProcessingStrategy {

    /**
     * Internally used correlation threshold.
     */
    double correlationThreshold = 0.5;
    
    /*
     * (non-Javadoc)
     * 
     * @see de.ugoe.cs.cpdp.IParameterizable#setParameter(java.lang.String)
     */
    @Override
    public void setParameter(String parameters) {
        if( parameters!=null && !parameters.equals("")) {
            correlationThreshold = Double.parseDouble(parameters);
        }
    }

    @Override
    public void apply(Instances testdata, SetUniqueList<Instances> traindataSet) {
        try {
            determineTopKAttributes(testdata, traindataSet);
        }
        catch (Exception e) {
            Console.printerr("Failure during metric selection: " + e.getMessage());
            throw new RuntimeException(e);
        }
    }

    private void determineTopKAttributes(Instances testdata, SetUniqueList<Instances> traindataSet) throws Exception {
        Integer[] counts = new Integer[traindataSet.get(0).numAttributes()-1];
        IntStream.range(0,counts.length).forEach(val -> counts[val] = 0);
        for( Instances traindata : traindataSet ) {
            J48 decisionTree = new J48();
            decisionTree.buildClassifier(traindata);
            int k=0;
            for( int j=0; j<traindata.numAttributes(); j++) {
                if(j!=traindata.classIndex()){
                    if( decisionTree.toString().contains(traindata.attribute(j).name()) ) {
                        counts[k] = counts[k]+1;
                    }
                    k++;
                }
            }
        }
        int[] topkIndex = new int[counts.length];
        IntStream.range(0,counts.length).forEach(val -> topkIndex[val] = val);
        SortUtils.quicksort(counts, topkIndex, true);
        
        // get CFSs for each training set
        List<Set<Integer>> cfsSets = new LinkedList<>();
        for( Instances traindata : traindataSet ) {
            AttributeSelection attsel = new AttributeSelection();
            CfsSubsetEval eval = new CfsSubsetEval();
            GreedyStepwise search = new GreedyStepwise();
            search.setSearchBackwards(true);
            attsel.setEvaluator(eval);
            attsel.setSearch(search);
            attsel.SelectAttributes(traindata);
            Set<Integer> cfsSet = new HashSet<>();
            for( int attr : attsel.selectedAttributes() ) {
                cfsSet.add(attr);
            }
            cfsSets.add(cfsSet);
        }
        
        double[] coverages = new double[topkIndex.length];
        for( Set<Integer> cfsSet : cfsSets ) {
            Set<Integer> topkSet = new HashSet<>();
            for( int k=0; k<topkIndex.length ; k++ ) {
{'fraction_non_alphanumeric': 0.1083323594717775, 'fraction_numerical': 0.007128666588757742, 'mean_word_length': 2.421831267493003, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4626839', 'n_tokens_mistral': 2445, 'n_tokens_neox': 2423, 'n_words': 579}