<commit_before>/*******************************************************************************
 * Copyright (c) 2012 <NAME>.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the GNU Lesser Public License v2.1
 * which accompanies this distribution, and is available at
 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 *
 * Contributors:
 *     <NAME>, <NAME> - initial API and implementation
 ******************************************************************************/
package org.coode.proximitymatrix;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.math.linear.MatrixIndexException;
import org.coode.distance.Distance;
import org.coode.distance.SparseMatrix;
import org.coode.distance.SparseMatrixFactory;
import org.coode.pair.Pair;
import org.coode.pair.SimplePair;
import org.coode.pair.filter.PairFilter;

public final class SimpleProximityMatrix<O> implements ProximityMatrix<O> {
    private final SparseMatrix delegate;
    private final List<O> objects = new ArrayList<O>();
    private final Comparator<? super Pair<O>> comparator;
    private double minimumDistance = Double.MAX_VALUE;
    private Pair<O> minimumDistancePair = null;
    private final PairFilter<O> filter;
    private final Map<O, Integer> objectIndex;

    public SimpleProximityMatrix(final Collection<? extends O> objects,
            final SparseMatrix distances, final PairFilter<O> filter,
            final Comparator<? super Pair<O>> comparator) {
        if (objects == null) {
            throw new NullPointerException("The object colleciton cannot be null");
        }
        if (objects.isEmpty()) {
            throw new IllegalArgumentException("The object collection cannot be empty");
        }
        if (distances == null) {
            throw new NullPointerException("The distances cannot be null");
        }
        if (distances.length() != objects.size()) {
            throw new IllegalArgumentException(String.format(
                    "The object collection size %d != distances dimension %d",
                    objects.size(), distances.length()));
        }
        if (filter == null) {
            throw new NullPointerException("The filter cannot be null");
        }
        if (comparator == null) {
            throw new NullPointerException("The comparator cannot be null");
        }
        this.comparator = comparator;
        this.objects.addAll(objects);
        this.filter = filter;
        this.objectIndex = new HashMap<O, Integer>();
        final int size = this.objects.size();
        for (int i = 0; i < size; i++) {
            O object = this.objects.get(i);
            this.objectIndex.put(object, i);
            for (int j = 0; j < size; j++) {
                O anotherObject = this.objects.get(j);
                double distanceValue = distances.get(i, j);
                SimplePair<O> pair = new SimplePair<O>(object, anotherObject);
                if (anotherObject != object
                        && filter.accept(object, anotherObject)
                        && (distanceValue < this.minimumDistance || distanceValue == this.minimumDistance
                                && this.minimumDistancePair != null
                                && this.comparator
                                        .compare(pair, this.minimumDistancePair) < 0)) {
                    this.minimumDistance = distanceValue;
                    this.minimumDistancePair = pair;
                }
            }
        }
        this.delegate = SparseMatrixFactory.create(distances);
    }

    public SimpleProximityMatrix(final Collection<? extends O> objects,
            final Distance<O> distance) {
        this(objects, distance, new PairFilter<O>() {
            public boolean accept(final O first, final O second) {
                return true;
            }
        }, new Comparator<Pair<O>>() {
            public int compare(final Pair<O> arg0, final Pair<O> arg1) {
                return arg0.hashCode() - arg1.hashCode();
            }
        });
    }

    public SimpleProximityMatrix(final Collection<? extends O> objects,
            final Distance<O> distance, final PairFilter<O> filter,
            final Comparator<? super Pair<O>> comparator) {
        if (objects == null) {
            throw new NullPointerException("The object colleciton cannot be null");
        }
        if (objects.isEmpty()) {
            throw new IllegalArgumentException("The object collection cannot be empty");
        }
        if (distance == null) {
            throw new NullPointerException("The distance cannot be null");
        }
        if (filter == null) {
            throw new NullPointerException("The filter cannot be null");
        }
        if (comparator == null) {
            throw new NullPointerException("The comparator cannot be null");
        }
        this.comparator = comparator;
        this.objects.addAll(objects);
        this.filter = filter;
        SparseMatrix distances = SparseMatrixFactory.create(objects.size());
        int i = 0;
        this.objectIndex = new HashMap<O, Integer>();
        for (O object : objects) {
            int j = 0;
            this.objectIndex.put(object, i);
            for (O anotherObject : objects) {
                double distanceValue = distance.getDistance(object, anotherObject);
                distances.set(i, j, distanceValue);
                SimplePair<O> pair = new SimplePair<O>(object, anotherObject);
                if (!anotherObject.equals(object)
                        && filter.accept(object, anotherObject)
                        && (distanceValue < this.minimumDistance || distanceValue == this.minimumDistance
                                && this.getMinimumDistancePair() != null
                                && this.comparator
                                        .compare(pair, this.minimumDistancePair) < 0)) {
                    this.minimumDistance = distanceValue;
                    this.minimumDistancePair = pair;
                }
                j++;
            }
            i++;
        }
        this.delegate = SparseMatrixFactory.create(distances);
    }

    public ProximityMatrix<O> reduce(final PairFilter<O> filter) {
        Set<O> reducedObjects = new HashSet<O>();
        Iterator<O> iterator = this.getObjects().iterator();
        boolean found = false;
        while (iterator.hasNext()) {
            Iterator<O> anotherIterator = this.getObjects().iterator();
            O object = iterator.next();
            while (!found && anotherIterator.hasNext()) {
                O anotherObject = anotherIterator.next();
                found = anotherObject != object && filter.accept(object, anotherObject);
            }
            if (found) {
                reducedObjects.add(object);
            }
            found = false;
        }
        SparseMatrix newDistances = SparseMatrixFactory.create(reducedObjects.size());
        List<O> list = new ArrayList<O>(reducedObjects);
        final int size = list.size();
        for (int i = 0; i < size; i++) {
            O a = list.get(i);
            for (int j = i + 1; j < size; j++) {
                O b = list.get(j);
                newDistances.set(i, j, a == b ? 0 : this.getDistance(a, b));
            }
        }
        return new SimpleProximityMatrix<O>(reducedObjects, newDistances,
                this.getFilter(), comparator);
    }

    public final int getRowIndex(final O o) {
        Integer index = this.objectIndex.get(o);
        return index == null ? -1 : index;
    }

    public final int getColumnIndex(final O o) {
        Integer index = this.objectIndex.get(o);
        return index == null ? -1 : index;
    }

    public Pair<O> getMinimumDistancePair() {
        return this.minimumDistancePair == null ? null : new SimplePair<O>(
                this.minimumDistancePair);
    }<commit_msg>Fixed a bug related to small proximity matrices that cannot be merged any further, so it would return an empty collection exception. <commit_after>/*******************************************************************************
 * Copyright (c) 2012 <NAME>.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the GNU Lesser Public License v2.1
 * which accompanies this distribution, and is available at
 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 *
 * Contributors:
 *     <NAME>, <NAME> - initial API and implementation
 ******************************************************************************/
package org.coode.proximitymatrix;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.math.linear.MatrixIndexException;
import org.coode.distance.Distance;
import org.coode.distance.SparseMatrix;
import org.coode.distance.SparseMatrixFactory;
import org.coode.pair.Pair;
import org.coode.pair.SimplePair;
import org.coode.pair.filter.PairFilter;

public final class SimpleProximityMatrix<O> implements ProximityMatrix<O> {
    private final SparseMatrix delegate;
    private final List<O> objects = new ArrayList<O>();
    private final Comparator<? super Pair<O>> comparator;
    private double minimumDistance = Double.MAX_VALUE;
    private Pair<O> minimumDistancePair = null;
    private final PairFilter<O> filter;
    private final Map<O, Integer> objectIndex;

    public SimpleProximityMatrix(final Collection<? extends O> objects,
            final SparseMatrix distances, final PairFilter<O> filter,
            final Comparator<? super Pair<O>> comparator) {
        if (objects == null) {
            throw new NullPointerException("The object colleciton cannot be null");
        }
        if (objects.isEmpty()) {
            throw new IllegalArgumentException("The object collection cannot be empty");
        }
        if (distances == null) {
            throw new NullPointerException("The distances cannot be null");
        }
        if (distances.length() != objects.size()) {
            throw new IllegalArgumentException(String.format(
                    "The object collection size %d != distances dimension %d",
                    objects.size(), distances.length()));
        }
        if (filter == null) {
            throw new NullPointerException("The filter cannot be null");
        }
        if (comparator == null) {
            throw new NullPointerException("The comparator cannot be null");
        }
        this.comparator = comparator;
        this.objects.addAll(objects);
        this.filter = filter;
        this.objectIndex = new HashMap<O, Integer>();
        final int size = this.objects.size();
        for (int i = 0; i < size; i++) {
            O object = this.objects.get(i);
            this.objectIndex.put(object, i);
            for (int j = 0; j < size; j++) {
                O anotherObject = this.objects.get(j);
                double distanceValue = distances.get(i, j);
                SimplePair<O> pair = new SimplePair<O>(object, anotherObject);
                if (anotherObject != object
                        && filter.accept(object, anotherObject)
                        && (distanceValue < this.minimumDistance || distanceValue == this.minimumDistance
                                && this.minimumDistancePair != null
                                && this.comparator
                                        .compare(pair, this.minimumDistancePair) < 0)) {
                    this.minimumDistance = distanceValue;
                    this.minimumDistancePair = pair;
                }
            }
        }
        this.delegate = SparseMatrixFactory.create(distances);
    }

    public SimpleProximityMatrix(final Collection<? extends O> objects,
            final Distance<O> distance) {
        this(objects, distance, new PairFilter<O>() {
            public boolean accept(final O first, final O second) {
                return true;
            }
        }, new Comparator<Pair<O>>() {
            public int compare(final Pair<O> arg0, final Pair<O> arg1) {
                return arg0.hashCode() - arg1.hashCode();
            }
        });
    }

    public SimpleProximityMatrix(final Collection<? extends O> objects,
            final Distance<O> distance, final PairFilter<O> filter,
            final Comparator<? super Pair<O>> comparator) {
        if (objects == null) {
            throw new NullPointerException("The object colleciton cannot be null");
        }
        if (objects.isEmpty()) {
            throw new IllegalArgumentException("The object collection cannot be empty");
        }
        if (distance == null) {
            throw new NullPointerException("The distance cannot be null");
        }
        if (filter == null) {
            throw new NullPointerException("The filter cannot be null");
        }
        if (comparator == null) {
            throw new NullPointerException("The comparator cannot be null");
        }
        this.comparator = comparator;
        this.objects.addAll(objects);
        this.filter = filter;
        SparseMatrix distances = SparseMatrixFactory.create(objects.size());
        int i = 0;
        this.objectIndex = new HashMap<O, Integer>();
        for (O object : objects) {
            int j = 0;
            this.objectIndex.put(object, i);
            for (O anotherObject : objects) {
                double distanceValue = distance.getDistance(object, anotherObject);
                distances.set(i, j, distanceValue);
                SimplePair<O> pair = new SimplePair<O>(object, anotherObject);
                if (!anotherObject.equals(object)
                        && filter.accept(object, anotherObject)
                        && (distanceValue < this.minimumDistance || distanceValue == this.minimumDistance
                                && this.getMinimumDistancePair() != null
                                && this.comparator
                                        .compare(pair, this.minimumDistancePair) < 0)) {
                    this.minimumDistance = distanceValue;
                    this.minimumDistancePair = pair;
                }
                j++;
            }
            i++;
        }
        this.delegate = SparseMatrixFactory.create(distances);
    }

    public ProximityMatrix<O> reduce(final PairFilter<O> filter) {
        Set<O> reducedObjects = new HashSet<O>();
        Iterator<O> iterator = this.getObjects().iterator();
        boolean found = false;
        while (iterator.hasNext()) {
            Iterator<O> anotherIterator = this.getObjects().iterator();
            O object = iterator.next();
            while (!found && anotherIterator.hasNext()) {
                O anotherObject = anotherIterator.next();
                found = anotherObject != object && filter.accept(object, anotherObject);
            }
            if (found) {
                reducedObjects.add(object);
            }
            found = false;
        }
        if(!reducedObjects.isEmpty()){
        	 SparseMatrix newDistances = SparseMatrixFactory.create(reducedObjects.size());
             List<O> list = new ArrayList<O>(reducedObjects);
             final int size = list.size();
             for (int i = 0; i < size; i++) {
                 O a = list.get(i);
                 for (int j = i + 1; j < size; j++) {
                     O b = list.get(j);
                     newDistances.set(i, j, a == b ? 0 : this.getDistance(a, b));
                 }
             }
             return new SimpleProximityMatrix<O>(reducedObjects, newDistances,
                     this.getFilter(), comparator);
        }
      //the proximity matrix cannot be reduced any more so return the same instance
        else{
        	 return this;
        }
       
    }

    public final int getRowIndex(final O o) {
        Integer index = this.objectIndex.get(o);
        return index == null ? -1 : index;
    }

    public final int getColumnIndex(final O o) {
        Integer index = this.objectIndex.get(o);
        return index == null ? -1 : index;
    }

    public Pair<O> getMinimumDistancePair() {
        return this.minimumDistancePair == null ? null : new SimplePair<O>(
                this.minimumDistancePair);
    }
{'fraction_non_alphanumeric': 0.11962583198416982, 'fraction_numerical': 0.0027582898602866225, 'mean_word_length': 2.050667642216938, 'pattern_counts': {'":': 0, '<': 109, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5454254', 'n_tokens_mistral': 4242, 'n_tokens_neox': 4057, 'n_words': 1167}