<commit_before>package de.lmu.ifi.dbs.elki.algorithm.outlier;

import java.util.BitSet;
import java.util.List;

import de.lmu.ifi.dbs.elki.data.RealVector;
import de.lmu.ifi.dbs.elki.database.Database;
import de.lmu.ifi.dbs.elki.distance.distancefunction.DimensionsSelectingEuclideanDistanceFunction;
import de.lmu.ifi.dbs.elki.normalization.NonNumericFeaturesException;
import de.lmu.ifi.dbs.elki.result.textwriter.TextWriteable;
import de.lmu.ifi.dbs.elki.result.textwriter.TextWriterStream;

/**
 * todo arthur comment
 *
 * @author <NAME>
 * @param <O> the type of DatabaseObjects handled by this Result
 */
public class SODModel<O extends RealVector<O, Double>> implements TextWriteable, Comparable<SODModel<O>> {
    private final DimensionsSelectingEuclideanDistanceFunction<O> DISTANCE_FUNCTION = new DimensionsSelectingEuclideanDistanceFunction<O>();

    private double[] centerValues;

    private O center;

    private double[] variances;

    private double expectationOfVariance;

    private BitSet weightVector;

    private double sod;
    
    /**
     * TODO: arthur comment
     * 
     * @param database
     * @param neighborhood
     * @param alpha
     * @param queryObject
     */
    public SODModel(Database<O> database, List<Integer> neighborhood, double alpha, O queryObject) {
        // TODO: store database link?
        centerValues = new double[database.dimensionality()];
        variances = new double[centerValues.length];
        for (Integer id : neighborhood) {
            O databaseObject = database.get(id);
            for (int d = 0; d < centerValues.length; d++) {
                centerValues[d] += databaseObject.getValue(d + 1);
            }
        }
        for (int d = 0; d < centerValues.length; d++) {
            centerValues[d] /= neighborhood.size();
        }        
        for (Integer id : neighborhood) {
            O databaseObject = database.get(id);
            for (int d = 0; d < centerValues.length; d++) {
                // distance
                double distance = centerValues[d] - databaseObject.getValue(d + 1);
                // variance
                variances[d] += distance * distance;
            }
        }
        expectationOfVariance = 0;
        for (int d = 0; d < variances.length; d++) {
            variances[d] /= neighborhood.size();
            expectationOfVariance += variances[d];
        }
        expectationOfVariance /= variances.length;
        weightVector = new BitSet(variances.length);
        for (int d = 0; d < variances.length; d++) {
            if (variances[d] < alpha * expectationOfVariance) {
                weightVector.set(d, true);
            }
        }
        DISTANCE_FUNCTION.setSelectedDimensions(weightVector);
        center = queryObject.newInstance(centerValues);
        sod = subspaceOutlierDegree(queryObject, center);
    }

    /**
     * TODO arthur comment
     * 
     * @param queryObject
     * @param center
     * @return sod value
     */
    public double subspaceOutlierDegree(O queryObject, O center) {
        double distance = DISTANCE_FUNCTION.distance(queryObject, center).getValue();
        distance /= weightVector.cardinality();
        return distance;
    }

    /**
     * Return the SOD of the point.
     * 
     * @return sod value
     */
    public double getSod() {
        return this.sod;
    }

    @Override<commit_msg>mini refactoring<commit_after>package de.lmu.ifi.dbs.elki.algorithm.outlier;

import java.util.BitSet;
import java.util.List;

import de.lmu.ifi.dbs.elki.data.RealVector;
import de.lmu.ifi.dbs.elki.database.Database;
import de.lmu.ifi.dbs.elki.distance.distancefunction.DimensionsSelectingEuclideanDistanceFunction;
import de.lmu.ifi.dbs.elki.normalization.NonNumericFeaturesException;
import de.lmu.ifi.dbs.elki.result.textwriter.TextWriteable;
import de.lmu.ifi.dbs.elki.result.textwriter.TextWriterStream;

/**
 * todo arthur comment
 *
 * @author <NAME>
 * @param <O> the type of DatabaseObjects handled by this Result
 */
public class SODModel<O extends RealVector<O, Double>> implements TextWriteable, Comparable<SODModel<O>> {
    private final DimensionsSelectingEuclideanDistanceFunction<O> DISTANCE_FUNCTION = new DimensionsSelectingEuclideanDistanceFunction<O>();

    private double[] centerValues;

    private O center;

    private double[] variances;

    private double expectationOfVariance;

    private BitSet weightVector;

    private double sod;
    
    /**
     * TODO: arthur comment
     * 
     * @param database
     * @param neighborhood
     * @param alpha
     * @param queryObject
     */
    public SODModel(Database<O> database, List<Integer> neighborhood, double alpha, O queryObject) {
        // TODO: store database link?
        centerValues = new double[database.dimensionality()];
        variances = new double[centerValues.length];
        for (Integer id : neighborhood) {
            O databaseObject = database.get(id);
            for (int d = 0; d < centerValues.length; d++) {
                centerValues[d] += databaseObject.getValue(d + 1);
            }
        }
        for (int d = 0; d < centerValues.length; d++) {
            centerValues[d] /= neighborhood.size();
        }        
        for (Integer id : neighborhood) {
            O databaseObject = database.get(id);
            for (int d = 0; d < centerValues.length; d++) {
                // distance
                double distance = centerValues[d] - databaseObject.getValue(d + 1);
                // variance
                variances[d] += distance * distance;
            }
        }
        expectationOfVariance = 0;
        for (int d = 0; d < variances.length; d++) {
            variances[d] /= neighborhood.size();
            expectationOfVariance += variances[d];
        }
        expectationOfVariance /= variances.length;
        weightVector = new BitSet(variances.length);
        for (int d = 0; d < variances.length; d++) {
            if (variances[d] < alpha * expectationOfVariance) {
                weightVector.set(d, true);
            }
        }
        center = queryObject.newInstance(centerValues);
        sod = subspaceOutlierDegree(queryObject, center, weightVector);
    }

    /**
     * TODO arthur comment
     * 
     * @param queryObject
     * @param center
     * @param weightVector
     * @return sod value
     */
    private double subspaceOutlierDegree(O queryObject, O center, BitSet weightVector) {
        DISTANCE_FUNCTION.setSelectedDimensions(weightVector);
        double distance = DISTANCE_FUNCTION.distance(queryObject, center).getValue();
        distance /= weightVector.cardinality();
        return distance;
    }

    /**
     * Return the SOD of the point.
     * 
     * @return sod value
     */
    public double getSod() {
        return this.sod;
    }

    @Override
{'fraction_non_alphanumeric': 0.10592354829296761, 'fraction_numerical': 0.0023344032681645753, 'mean_word_length': 2.412145345943255, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5008535', 'n_tokens_mistral': 2012, 'n_tokens_neox': 1910, 'n_words': 449}