<commit_before>package org.intermine.dataloader;

/*
 * Copyright (C) 2002-2007 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.util.Comparator;
import java.util.List;
import java.util.Map;

import static org.intermine.dataloader.IntegrationWriterAbstractImpl.SKELETON;
import org.intermine.metadata.ClassDescriptor;
import org.intermine.metadata.FieldDescriptor;
import org.intermine.model.InterMineObject;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.objectstore.proxy.ProxyReference;
import org.intermine.util.IntPresentSet;
import org.intermine.util.TypeUtil;

import org.apache.log4j.Logger;

/**
 * Comparator, that compares two InterMineObjects, with reference to a particular
 * FieldDescriptor for priority.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class SourcePriorityComparator implements Comparator
{
    private static final Logger LOG = Logger.getLogger(SourcePriorityComparator.class);

    private DataTracker dataTracker;
    private FieldDescriptor field;
    private Source def;
    private InterMineObject defObj;
    private IntPresentSet dbIdsStored;
    private IntegrationWriterAbstractImpl iw;
    private Source source, skelSource;

    /**
     * Constructs a new Comparator for comparing objects for priority for a given field.
     *
     * @param dataTracker the data tracker
     * @param field the FieldDescriptor the comparison is for
     * @param def the default Source
     * @param defObj a InterMineObject that came from a data source, not from the destination
     * objectstore, and should be associated with the default source
     * @param iw the IntegrationWriter creating this comparator
     * @param source the main source, as passed to iw.store
     * @param skelSource the skeleton source, as passed to iw.store
     * @param dbIdsStored the set of IDs stored in this dataloader run - improves error messages
     */
    public SourcePriorityComparator(DataTracker dataTracker, FieldDescriptor field,
            Source def, InterMineObject defObj, IntPresentSet dbIdsStored,
            IntegrationWriterAbstractImpl iw, Source source, Source skelSource) {
        this.dataTracker = dataTracker;
        this.field = field;
        this.def = def;
        this.defObj = defObj;
        this.dbIdsStored = dbIdsStored;
        this.iw = iw;
        this.source = source;
        this.skelSource = skelSource;
    }

    /**
     * Compares two objects. These objects must both be InterMineObjects.
     *
     * @param o1 the first object
     * @param o2 the second object
     * @return a negative integer, zero, or a positive integer as the first argument is less than,
     * equal to, or greater than the second
     * @throws ClassCastException if either of the two objects is not a InterMineObject
     * @throws RuntimeException if an error occurs in the underlying data tracking objectstore
     */
    public int compare(Object o1, Object o2) {
        if ((o1 instanceof InterMineObject) && (o2 instanceof InterMineObject)) {
            InterMineObject f1 = (InterMineObject) o1;
            InterMineObject f2 = (InterMineObject) o2;
            Source source1 = null;
            Source source2 = null;
            Object value1, value2;
            try {
                value1 = TypeUtil.getFieldProxy(f1, field.getName());
                value2 = TypeUtil.getFieldProxy(f2, field.getName());
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
            ClassDescriptor cld = field.getClassDescriptor();
            String cldName = TypeUtil.unqualifiedName(cld.getName());
            Map descriptorSources = DataLoaderHelper.getDescriptors(cld.getModel());
            List srcs = (List) descriptorSources.get(cldName + "." + field.getName());
            if (srcs == null) {
                srcs = (List) descriptorSources.get(cldName);
            }
            if (srcs != null) {
                if (f1 == defObj) {
                    source1 = def;
                } else {
                    source1 = dataTracker.getSource(f1.getId(), field.getName());
                }
                if (f2 == defObj) {
                    source2 = def;
                } else {
                    source2 = dataTracker.getSource(f2.getId(), field.getName());
                }
                if (source1 == null) {
                    throw new IllegalArgumentException("Object o1 is not in the data"
                            + " tracking system; o1 = \"" + o1 + "\", o2 = \"" + o2
                            + "\" for field \"" + field.getName() + "\"");
                }
                if (source2 == null) {
                    throw new IllegalArgumentException("Object o2 is not in the data"
                            + " tracking system; o1 = \"" + o1 + "\", o2 = \"" + o2
                            + "\" for field \"" + field.getName() + "\"");
                }
                if (!srcs.contains(source1.getName())) {
                    throw new IllegalArgumentException("Priority configured for " + cldName + "."
                            + field.getName() + " does not include source " + source1.getName());
                }
                if (!srcs.contains(source2.getName())) {
                    throw new IllegalArgumentException("Priority configured for " + cldName + "."
                            + field.getName() + " does not include source " + source2.getName());
                }
                int retval = srcs.indexOf(source2.getName()) - srcs.indexOf(source1.getName());
                if ((retval == 0) && (!o1.equals(o2)) && (!source1.getSkeleton())) {
                    String errMessage = "Unequivalent objects have the same"
                        + " non-skeleton Source; o1 = \"" + o1 + "\" ("
                        + (o1 == defObj ? "from source" : (dbIdsStored.contains(f1.getId())
                                    ? "stored in this run" : "from database")) + "), o2 = \"" + o2
                        + "\"(" + (o2 == defObj ? "from source" : (dbIdsStored.contains(f2.getId())
                                    ? "stored in this run" : "from database")) + "), source1 = \""
                        + source1 + "\", source2 = \"" + source2 + "\" for field \""
                        + field.getName() + "\"";
                    LOG.error(errMessage);
                    throw new IllegalArgumentException(errMessage);
                }<commit_msg>Fix to improvement to SourcePriorityComparator - should behave more as before now<commit_after>package org.intermine.dataloader;

/*
 * Copyright (C) 2002-2007 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.util.Comparator;
import java.util.List;
import java.util.Map;

import static org.intermine.dataloader.IntegrationWriterAbstractImpl.SKELETON;
import org.intermine.metadata.ClassDescriptor;
import org.intermine.metadata.FieldDescriptor;
import org.intermine.model.InterMineObject;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.objectstore.proxy.ProxyReference;
import org.intermine.util.IntPresentSet;
import org.intermine.util.TypeUtil;

import org.apache.log4j.Logger;

/**
 * Comparator, that compares two InterMineObjects, with reference to a particular
 * FieldDescriptor for priority.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class SourcePriorityComparator implements Comparator
{
    private static final Logger LOG = Logger.getLogger(SourcePriorityComparator.class);

    private DataTracker dataTracker;
    private FieldDescriptor field;
    private Source def;
    private InterMineObject defObj;
    private IntPresentSet dbIdsStored;
    private IntegrationWriterAbstractImpl iw;
    private Source source, skelSource;

    /**
     * Constructs a new Comparator for comparing objects for priority for a given field.
     *
     * @param dataTracker the data tracker
     * @param field the FieldDescriptor the comparison is for
     * @param def the default Source
     * @param defObj a InterMineObject that came from a data source, not from the destination
     * objectstore, and should be associated with the default source
     * @param iw the IntegrationWriter creating this comparator
     * @param source the main source, as passed to iw.store
     * @param skelSource the skeleton source, as passed to iw.store
     * @param dbIdsStored the set of IDs stored in this dataloader run - improves error messages
     */
    public SourcePriorityComparator(DataTracker dataTracker, FieldDescriptor field,
            Source def, InterMineObject defObj, IntPresentSet dbIdsStored,
            IntegrationWriterAbstractImpl iw, Source source, Source skelSource) {
        this.dataTracker = dataTracker;
        this.field = field;
        this.def = def;
        this.defObj = defObj;
        this.dbIdsStored = dbIdsStored;
        this.iw = iw;
        this.source = source;
        this.skelSource = skelSource;
    }

    /**
     * Compares two objects. These objects must both be InterMineObjects.
     *
     * @param o1 the first object
     * @param o2 the second object
     * @return a negative integer, zero, or a positive integer as the first argument is less than,
     * equal to, or greater than the second
     * @throws ClassCastException if either of the two objects is not a InterMineObject
     * @throws RuntimeException if an error occurs in the underlying data tracking objectstore
     */
    public int compare(Object o1, Object o2) {
        if ((o1 instanceof InterMineObject) && (o2 instanceof InterMineObject)) {
            InterMineObject f1 = (InterMineObject) o1;
            InterMineObject f2 = (InterMineObject) o2;
            Source source1 = null;
            Source source2 = null;
            Object value1, value2;
            try {
                value1 = TypeUtil.getFieldProxy(f1, field.getName());
                value2 = TypeUtil.getFieldProxy(f2, field.getName());
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
            ClassDescriptor cld = field.getClassDescriptor();
            String cldName = TypeUtil.unqualifiedName(cld.getName());
            Map descriptorSources = DataLoaderHelper.getDescriptors(cld.getModel());
            List srcs = (List) descriptorSources.get(cldName + "." + field.getName());
            if (srcs == null) {
                srcs = (List) descriptorSources.get(cldName);
            }
            if (srcs != null) {
                if (f1 == defObj) {
                    source1 = def;
                } else {
                    source1 = dataTracker.getSource(f1.getId(), field.getName());
                }
                if (f2 == defObj) {
                    source2 = def;
                } else {
                    source2 = dataTracker.getSource(f2.getId(), field.getName());
                }
                if (source1 == null) {
                    throw new IllegalArgumentException("Object o1 is not in the data"
                            + " tracking system; o1 = \"" + o1 + "\", o2 = \"" + o2
                            + "\" for field \"" + field.getName() + "\"");
                }
                if (source2 == null) {
                    throw new IllegalArgumentException("Object o2 is not in the data"
                            + " tracking system; o1 = \"" + o1 + "\", o2 = \"" + o2
                            + "\" for field \"" + field.getName() + "\"");
                }
                if (source1.getName().equals(source2.getName())) {
                    if (source1.getSkeleton() && (!source2.getSkeleton())) {
                        return -1;
                    } else if (source2.getSkeleton() && (!source1.getSkeleton())) {
                        return 1;
                    } else {
                        if ((!o1.equals(o2)) && (!source1.getSkeleton())) {
                            String errMessage = "Unequivalent objects have the same"
                                + " non-skeleton Source; o1 = \"" + o1 + "\" ("
                                + (o1 == defObj ? "from source" : (dbIdsStored.contains(f1.getId())
                                            ? "stored in this run" : "from database"))
                                + "), o2 = \"" + o2 + "\"(" + (o2 == defObj ? "from source"
                                        : (dbIdsStored.contains(f2.getId())
                                            ? "stored in this run" : "from database"))
                                + "), source1 = \"" + source1 + "\", source2 = \"" + source2
                                + "\" for field \"" + field.getName() + "\"";
                            LOG.error(errMessage);
                            throw new IllegalArgumentException(errMessage);
                        }
                        return 0;
                    }
                } 
                String errorMessage = null;
                if (!srcs.contains(source1.getName())) {
                    errorMessage = "Priority configured for " + cldName + "." + field.getName()
                        + " does not include source " + source1.getName();
                }
                if (!srcs.contains(source2.getName())) {
                    errorMessage = "Priority configured for " + cldName + "." + field.getName()
                        + " does not include source " + source2.getName();
                }
                if (errorMessage != null) {
                    if ((value1 == null) && (value2 == null)) {
                        return (f1 == defObj ? 1 : -1);
                    }
                    if (value1 == null) {
                        return -1;
                    }
                    if (value2 == null) {
                        return 1;
                    }
                    LOG.error(errorMessage);
                    throw new IllegalArgumentException(errorMessage);
                }
                int retval = srcs.indexOf(source2.getName()) - srcs.indexOf(source1.getName());
{'fraction_non_alphanumeric': 0.09866519884408972, 'fraction_numerical': 0.010664648410623366, 'mean_word_length': 1.8228782287822878, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4744132', 'n_tokens_mistral': 3796, 'n_tokens_neox': 3663, 'n_words': 1084}