<commit_before>package org.intermine.bio.postprocess;

/*
 * Copyright (C) 2002-2005 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */
import java.util.Iterator;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
import java.util.HashSet;
import java.util.List;
import java.util.ArrayList;
import java.util.Collections;

import org.intermine.objectstore.query.*;
import org.intermine.objectstore.ObjectStore;
import org.intermine.objectstore.ObjectStoreWriter;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.objectstore.intermine.ObjectStoreInterMineImpl;
import org.intermine.util.DynamicUtil;
import org.intermine.util.TypeUtil;
import org.intermine.objectstore.query.iql.IqlQuery;

import org.intermine.model.InterMineObject;
import org.flymine.model.genomic.*;
import org.apache.log4j.Logger;


/**
 * Calculate additional mappings between annotation after loading into genomic ObjectStore.
 * Currently designed to cope with situation after loading ensembl, may need to change
 * as other annotation is loaded.  New Locations (and updated BioEntities) are stored
 * back in originating ObjectStore.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class CalculateLocations
{
    private static final Logger LOG = Logger.getLogger(CalculateLocations.class);

    protected ObjectStoreWriter osw;
    protected ObjectStore os;
    protected Map chrById = null;
    protected Map bandToChr = new HashMap();
    protected Map chrToBand = new HashMap();
    protected Map chrToSc = new HashMap();
    protected Map scToChr = new HashMap();
    protected Map contigToSc = new HashMap();
    protected Map contigToChr = new HashMap();

    /**
     * Create a new CalculateLocations object from an ObjectStoreWriter
     * @param osw writer on genomic ObjectStore
     */
    public CalculateLocations(ObjectStoreWriter osw) {
        this.osw = osw;
        this.os = osw.getObjectStore();
    }

    /**
     * Turn Location objects that should be partial into PartialLocations.  Locations are only
     * converted if they go to the start/end of the object BioEntity and there is at one other
     * location that refers to the same subject.
     * eg.
     *   object1  ---------++------------ object2
     *                |loc1||loc2|
     *       subject  +----------+
     * or
     *   object1  ---------++----++------------ object2
     *                |loc1||loc2||loc3|
     *       subject  +----------------+
     *
     * Locations are ignored that happen to go to the start or end but there is only one Location
     * that refers to the subject:
     *   object   ---------+
     *                |loc1|
     *   subject      +----+
     * @throws Exception if anything goes wrong
     */
    public void fixPartials() throws Exception {
        fixPartials(Contig.class, Exon.class);
    }

    /**
     * Fix the Locations that connect objectCls and subjectCls objects.
     */
    private void fixPartials(Class objectCls, Class subjectCls) throws Exception {
        Results results = PostProcessUtil.findLocationAndObjects(os, objectCls, subjectCls, true);
        results.setBatchSize(500);

        osw.beginTransaction();

        Iterator resIter = results.iterator();

        Set batch = new HashSet();

        int previousSubjectId = -1;

        while (resIter.hasNext()) {
            ResultsRow rr = (ResultsRow) resIter.next();
            BioEntity subject = (BioEntity) rr.get(1);
            if (subject.getId().intValue() != previousSubjectId && batch.size() > 0) {
                fixPartialBatch(batch);
                batch = new HashSet();
            }

            batch.add(rr);
            previousSubjectId = subject.getId().intValue();
        }

        if (previousSubjectId != -1 && batch.size() > 0) {
            fixPartialBatch(batch);
        }

        osw.commitTransaction();
    }

    /**
     * The argument should be a Set of ResultsRow objects returned by findLocations().  All of the
     * Locations in the ResultsRow should have the same Subject.
     */
    private void fixPartialBatch(Set batch) throws Exception {
        if (batch.size() < 2) {
            // if the object doesn't have two location objects the locations can't be partial
            return;
        }

        LOG.info("processing partial batch size " + batch.size());

        Location startLocation = null;
        Location endLocation = null;
        // Locations that are partial at the start and the end
        Set veryPartialLocations = new HashSet();

        int subjectLengthSoFar = -1;

        Iterator batchIter = batch.iterator();
        while (batchIter.hasNext()) {
            ResultsRow rr = (ResultsRow) batchIter.next();
            Integer objectId = (Integer) rr.get(0);
            BioEntity object = (BioEntity) os.getObjectById(objectId);
            Location location = (Location) rr.get(2);

            try {
                Object objectLengthField = TypeUtil.getFieldValue(object, "length");
                int objectLength = -1;
                if (objectLengthField instanceof Integer) {
                    objectLength = ((Integer) objectLengthField).intValue();
               } else {
                    LOG.error("Object with ID: "
                              + object.getId() + " has no Integer length field");
                    continue;
                }

                if (location.getEnd().intValue() == objectLength
                    && location.getStart().intValue() == 1) {
                    // both ends are partial - fix last
                    veryPartialLocations.add(location);
                }

                if (location.getStart().intValue() == 1
                    && location.getEnd().intValue() != objectLength) {
                    endLocation = location;
                }

                if (location.getEnd().intValue() == objectLength
                    && location.getStart().intValue() != 1) {
                    PartialLocation pl =
                        (PartialLocation) cloneInterMineObject(location, PartialLocation.class);
                    pl.setStartIsPartial(Boolean.FALSE);
                    pl.setEndIsPartial(Boolean.TRUE);
                    pl.setSubjectStart(new Integer(1));
                    subjectLengthSoFar =
                        location.getEnd().intValue() - location.getStart().intValue() + 1;
                    pl.setSubjectEnd(new Integer(subjectLengthSoFar));
                    startLocation = pl;
                    osw.store(pl);
                }
            } catch (IllegalAccessException e) {
                LOG.error("Object with ID: " + object.getId()
                          + " has no Integer length field");
            }
        }

        Iterator veryPartialLocationsIterator = veryPartialLocations.iterator();

        while (veryPartialLocationsIterator.hasNext()) {
            Location location = (Location) veryPartialLocationsIterator.next();

            int thisObjectLength =
                location.getEnd().intValue() - location.getStart().intValue() + 1;

            PartialLocation pl =
                (PartialLocation) cloneInterMineObject(location, PartialLocation.class);
            pl.setStartIsPartial(Boolean.TRUE);
            pl.setEndIsPartial(Boolean.TRUE);
            pl.setSubjectStart(new Integer(subjectLengthSoFar + 1));
            pl.setSubjectEnd(new Integer(subjectLengthSoFar + thisObjectLength));
            osw.store(pl);
            subjectLengthSoFar += thisObjectLength;
        }

        if (endLocation == null) {
            LOG.error("endLocation is null - startLocation: " + startLocation
                      + "  veryPartialLocations: " + veryPartialLocations);
        } else {
            PartialLocation pl =
                (PartialLocation) cloneInterMineObject(endLocation, PartialLocation.class);

            pl.setStartIsPartial(Boolean.TRUE);
            pl.setEndIsPartial(Boolean.FALSE);
            pl.setSubjectStart(new Integer(subjectLengthSoFar + 1));
            int newEndPos = pl.getSubjectStart().intValue()
                + (pl.getEnd().intValue() - pl.getStart().intValue());
            pl.setSubjectEnd(new Integer(newEndPos));
            osw.store(pl);
        }

    }


    // TODO 1) support pos/neg strand
    //      2) store chromosomes in id map -> avoid getObjectById
    //      3) map from chromosome to children -> less overlap comparasons
    //      4) check if a Location already exists?
    //      5) evidence collection for locations

    /**
     * Create new Location objects where required:
     *
     *  | = Location that should exist   ( = Location to create
     *
     * Chromosome
     *   | | ( (
     *   ChromosomeBand
     *     ( ( (
     *     Supercontig
     *       | (
     *       Contig
     *         |
     *         *features*
     *
     * @throws Exception if anything goes wrong
     */
    public void createLocations() throws Exception {
        osw.beginTransaction();

        // 0. Hold Chromosomes in map by id
        chrById = makeChromosomeMap();

        // 1. Find and hold locations of ChromosomeBands on Chromsomes
        makeChromosomeBandLocations();

        // 2. Find and hold locations of Supercontigs on Chromosomes
        //    Create locations of Supercontigs on ChromosomeBands
        makeSupercontigLocations();

        // 3. hold offsets of Contigs on Supercontigs
        //    create locations Contig->ChromosomeBand, Contig->Chromosome
        makeContigLocations();

        // 4. For all BioEntities located on Contigs compute other offsets on all parents
        Results results =
            PostProcessUtil.findLocationAndObjects(os, Contig.class, BioEntity.class, false);
        results.setBatchSize(500);

        Iterator resIter = results.iterator();

        // create map ChromsomeBands to avoid calling getObjectById
        // need to keep running query after each commit transaction
        Map idBands = new HashMap();
        Iterator bandIter = PostProcessUtil.selectObjectsOfClass(os, ChromosomeBand.class);
        while (bandIter.hasNext()) {
            ChromosomeBand band = (ChromosomeBand) bandIter.next();
            idBands.put(band.getId(), band);
        }
        LOG.info("built ChromosomeBand id map, size = " + idBands.keySet().size());

        // create map of Supercontigs to avoid calling getObjectById
        // need to keep running query after each commit transaction
        Map idScs = new HashMap();
        Iterator scIter = PostProcessUtil.selectObjectsOfClass(os, Supercontig.class);
        while (scIter.hasNext()) {
            Supercontig sc = (Supercontig) scIter.next();
            idScs.put(sc.getId(), sc);
        }
        LOG.info("built Supercontig id map, size = " + idScs.keySet().size());

        // maps from BioEntity to Location
        Map partialsOnChromosomes = new HashMap();
        Map partialsOnSupercontigs = new HashMap();
        Map partialsOnChromosomeBands = new HashMap();

        int i = 0, j = 0, k = 0;
        long start = System.currentTimeMillis();
        while (resIter.hasNext()) {
            i++;
            ResultsRow rr = (ResultsRow) resIter.next();
            Location locBioOnContig = (Location) rr.get(2);
            Integer contigId = (Integer) rr.get(0);
            BioEntity bio = (BioEntity) rr.get(1);
            SimpleLoc bioOnContig = new SimpleLoc(contigId.intValue(),
                                                  bio.getId().intValue(),
                                                  locBioOnContig);

            // first create location of feature on Chromosome
            SimpleLoc contigOnChr = (SimpleLoc) contigToChr.get(contigId);

            Chromosome chr = (Chromosome)
                chrById.get(new Integer(contigOnChr.getParentId()));
            Location bioOnChrLoc =
                createTransformedLocation(contigOnChr, bioOnContig, chr, bio);

            if (locBioOnContig instanceof PartialLocation) {
                addToMapOfLists(partialsOnChromosomes, bio, bioOnChrLoc);
            } else {
                osw.store(bioOnChrLoc);
            }

            SimpleLoc bioOnChr = new SimpleLoc(chr.getId().intValue(),
                                               bio.getId().intValue(),
                                               bioOnChrLoc);

            // create location of feature on Supercontig
            Set scs = (Set) chrToSc.get(chr.getId());
            if (scs != null) {
                Iterator iter = scs.iterator();
                while (iter.hasNext()) {
                    SimpleLoc scOnChr = (SimpleLoc) iter.next();
                    if (overlap(scOnChr, bioOnChr)) {
                        Supercontig sc = (Supercontig) idScs.get(new Integer(scOnChr.getChildId()));
                        Location bioOnScLoc = createLocation(sc, scOnChr, bio, bioOnChr);

                        if (bioOnScLoc instanceof PartialLocation) {
                            addToMapOfLists(partialsOnSupercontigs, bio, bioOnScLoc);
                        } else {
                            osw.store(bioOnScLoc);
                        }
                        j++;
                    }
                }
            }

            // create location of feature on ChromosomeBand
            Set bands = (Set) chrToBand.get(chr.getId());
            if (bands != null) {
                Iterator iter = bands.iterator();
                while (iter.hasNext()) {
                    SimpleLoc bandOnChr = (SimpleLoc) iter.next();
                    if (overlap(bandOnChr, bioOnChr)) {
                        ChromosomeBand band = (ChromosomeBand)
                            idBands.get(new Integer(bandOnChr.getChildId()));
                        Location bioOnBandLoc = createLocation(band, bandOnChr, bio, bioOnChr);

                        if (bioOnBandLoc instanceof PartialLocation) {
                            addToMapOfLists(partialsOnChromosomeBands, bio, bioOnBandLoc);
                        } else {
                            osw.store(bioOnBandLoc);
                        }
                        k++;
                    }
                }
            }
            if (i % 100 == 0) {
                long now = System.currentTimeMillis();
                LOG.info("Created " + i + " Chromosome, "
                         + j + " SuperContig locations and "
                         + k + " ChromosomeBand locations (avg = "
                         + ((60000L * i) / (now - start)) + " per minute)");
            }
        }

        chrById = null;
        bandToChr = null;
        chrToBand = null;
        chrToSc = null;
        scToChr = null;
        contigToSc = null;
        contigToChr = null;

        // process partials Locations
        processPartials(partialsOnChromosomes);
        processPartials(partialsOnSupercontigs);
        processPartials(partialsOnChromosomeBands);

        osw.commitTransaction();

        LOG.info("Stored " + i + " Locations between features and Chromosome.");
        LOG.info("Stored " + j + " Locations between features and Supercontig.");
        LOG.info("Stored " + k + " Locations between features and ChromosomeBand.");
    }

    /**
     * Create OverlapRelation objects for all overlapping LocatedSequenceFeatures by querying
     * objects that are located on chromosomes and overlap.
     * @param classNamesToIgnore a comma separated list of the names of those classes that should be
     * ignored when searching for overlaps.  Sub classes to these classes are ignored too
     * @param ignoreSelfMatches if true, don't create OverlapRelations between two objects of the
     * same class.
     * @throws Exception if anything goes wrong
     */
    public void createOverlapRelations(List classNamesToIgnore, boolean ignoreSelfMatches)
         throws Exception {
        osw.beginTransaction();
        Map chromosomeMap = makeChromosomeMap();
        Iterator chromosomeIdIter = chromosomeMap.keySet().iterator();
        while (chromosomeIdIter.hasNext()) {
            Integer id = (Integer) chromosomeIdIter.next();
            createSubjectOverlapRelations((Chromosome) chromosomeMap.get(id), classNamesToIgnore,
                                          ignoreSelfMatches);
        }
        osw.commitTransaction();
    }

    /**
     * Create OverlapRelation objects for locations that have the given subject.
     * @param classNamesToIgnore a comma separated list of the names of those classes that should be
     * ignored when searching for overlaps.  Sub classes to these classes are ignored too
     * @param ignoreSelfMatches if true, don't create OverlapRelations between two objects of the
     * same class.
     */
    private void createSubjectOverlapRelations(Chromosome subject, List classNamesToIgnore,
                                               boolean ignoreSelfMatches)
        throws Exception {
        LOG.info("Creating overlaps for " + subject + ", identifier: "
                 + subject.getIdentifier());

        Iterator overlapIterator =
            OverlapUtil.findOverlaps(os, subject, classNamesToIgnore, ignoreSelfMatches);

        int count = 0;

        while (overlapIterator.hasNext()) {
            Location overlapLocations[] = (Location[]) overlapIterator.next();

            Location location1 = overlapLocations[0];
            Location location2 = overlapLocations[1];

            InterMineObject lsf1 = location1.proxGetSubject();
            InterMineObject lsf2 = location2.proxGetSubject();

            OverlapRelation overlapRelation = (OverlapRelation)
                DynamicUtil.createObject(Collections.singleton(OverlapRelation.class));
            overlapRelation.addBioEntities(lsf1);
            overlapRelation.addBioEntities(lsf2);

            ++count;

            osw.store(overlapRelation);
        }

        LOG.info("Stored " + count + " overlaps for " + subject + ", identifier: "
                 + subject.getIdentifier());
    }

    /**
     * Put key and value in the given map.  The values of the map are List, to which the new value
     * is appended.  The Lists are created if missing.
     */
    private void addToMapOfLists(Map map, Object key, Object value) {
        if (map.get(key) != null) {<commit_msg>Fix compile<commit_after>package org.intermine.bio.postprocess;

/*
 * Copyright (C) 2002-2005 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */
import java.util.Iterator;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
import java.util.HashSet;
import java.util.List;
import java.util.ArrayList;
import java.util.Collections;

import org.intermine.objectstore.query.*;
import org.intermine.objectstore.ObjectStore;
import org.intermine.objectstore.ObjectStoreWriter;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.objectstore.intermine.ObjectStoreInterMineImpl;
import org.intermine.util.DynamicUtil;
import org.intermine.util.TypeUtil;
import org.intermine.objectstore.query.iql.IqlQuery;

import org.intermine.model.InterMineObject;
import org.flymine.model.genomic.*;
import org.apache.log4j.Logger;


/**
 * Calculate additional mappings between annotation after loading into genomic ObjectStore.
 * Currently designed to cope with situation after loading ensembl, may need to change
 * as other annotation is loaded.  New Locations (and updated BioEntities) are stored
 * back in originating ObjectStore.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class CalculateLocations
{
    private static final Logger LOG = Logger.getLogger(CalculateLocations.class);

    protected ObjectStoreWriter osw;
    protected ObjectStore os;
    protected Map chrById = null;
    protected Map bandToChr = new HashMap();
    protected Map chrToBand = new HashMap();
    protected Map chrToSc = new HashMap();
    protected Map scToChr = new HashMap();
    protected Map contigToSc = new HashMap();
    protected Map contigToChr = new HashMap();

    /**
     * Create a new CalculateLocations object from an ObjectStoreWriter
     * @param osw writer on genomic ObjectStore
     */
    public CalculateLocations(ObjectStoreWriter osw) {
        this.osw = osw;
        this.os = osw.getObjectStore();
    }

    /**
     * Turn Location objects that should be partial into PartialLocations.  Locations are only
     * converted if they go to the start/end of the object BioEntity and there is at one other
     * location that refers to the same subject.
     * eg.
     *   object1  ---------++------------ object2
     *                |loc1||loc2|
     *       subject  +----------+
     * or
     *   object1  ---------++----++------------ object2
     *                |loc1||loc2||loc3|
     *       subject  +----------------+
     *
     * Locations are ignored that happen to go to the start or end but there is only one Location
     * that refers to the subject:
     *   object   ---------+
     *                |loc1|
     *   subject      +----+
     * @throws Exception if anything goes wrong
     */
    public void fixPartials() throws Exception {
        fixPartials(Contig.class, Exon.class);
    }

    /**
     * Fix the Locations that connect objectCls and subjectCls objects.
     */
    private void fixPartials(Class objectCls, Class subjectCls) throws Exception {
        Results results = PostProcessUtil.findLocationAndObjects(os, objectCls, subjectCls, true);
        results.setBatchSize(500);

        osw.beginTransaction();

        Iterator resIter = results.iterator();

        Set batch = new HashSet();

        int previousSubjectId = -1;

        while (resIter.hasNext()) {
            ResultsRow rr = (ResultsRow) resIter.next();
            BioEntity subject = (BioEntity) rr.get(1);
            if (subject.getId().intValue() != previousSubjectId && batch.size() > 0) {
                fixPartialBatch(batch);
                batch = new HashSet();
            }

            batch.add(rr);
            previousSubjectId = subject.getId().intValue();
        }

        if (previousSubjectId != -1 && batch.size() > 0) {
            fixPartialBatch(batch);
        }

        osw.commitTransaction();
    }

    /**
     * The argument should be a Set of ResultsRow objects returned by findLocations().  All of the
     * Locations in the ResultsRow should have the same Subject.
     */
    private void fixPartialBatch(Set batch) throws Exception {
        if (batch.size() < 2) {
            // if the object doesn't have two location objects the locations can't be partial
            return;
        }

        LOG.info("processing partial batch size " + batch.size());

        Location startLocation = null;
        Location endLocation = null;
        // Locations that are partial at the start and the end
        Set veryPartialLocations = new HashSet();

        int subjectLengthSoFar = -1;

        Iterator batchIter = batch.iterator();
        while (batchIter.hasNext()) {
            ResultsRow rr = (ResultsRow) batchIter.next();
            Integer objectId = (Integer) rr.get(0);
            BioEntity object = (BioEntity) os.getObjectById(objectId);
            Location location = (Location) rr.get(2);

            try {
                Object objectLengthField = TypeUtil.getFieldValue(object, "length");
                int objectLength = -1;
                if (objectLengthField instanceof Integer) {
                    objectLength = ((Integer) objectLengthField).intValue();
               } else {
                    LOG.error("Object with ID: "
                              + object.getId() + " has no Integer length field");
                    continue;
                }

                if (location.getEnd().intValue() == objectLength
                    && location.getStart().intValue() == 1) {
                    // both ends are partial - fix last
                    veryPartialLocations.add(location);
                }

                if (location.getStart().intValue() == 1
                    && location.getEnd().intValue() != objectLength) {
                    endLocation = location;
                }

                if (location.getEnd().intValue() == objectLength
                    && location.getStart().intValue() != 1) {
                    PartialLocation pl =
                        (PartialLocation) cloneInterMineObject(location, PartialLocation.class);
                    pl.setStartIsPartial(Boolean.FALSE);
                    pl.setEndIsPartial(Boolean.TRUE);
                    pl.setSubjectStart(new Integer(1));
                    subjectLengthSoFar =
                        location.getEnd().intValue() - location.getStart().intValue() + 1;
                    pl.setSubjectEnd(new Integer(subjectLengthSoFar));
                    startLocation = pl;
                    osw.store(pl);
                }
            } catch (IllegalAccessException e) {
                LOG.error("Object with ID: " + object.getId()
                          + " has no Integer length field");
            }
        }

        Iterator veryPartialLocationsIterator = veryPartialLocations.iterator();

        while (veryPartialLocationsIterator.hasNext()) {
            Location location = (Location) veryPartialLocationsIterator.next();

            int thisObjectLength =
                location.getEnd().intValue() - location.getStart().intValue() + 1;

            PartialLocation pl =
                (PartialLocation) cloneInterMineObject(location, PartialLocation.class);
            pl.setStartIsPartial(Boolean.TRUE);
            pl.setEndIsPartial(Boolean.TRUE);
            pl.setSubjectStart(new Integer(subjectLengthSoFar + 1));
            pl.setSubjectEnd(new Integer(subjectLengthSoFar + thisObjectLength));
            osw.store(pl);
            subjectLengthSoFar += thisObjectLength;
        }

        if (endLocation == null) {
            LOG.error("endLocation is null - startLocation: " + startLocation
                      + "  veryPartialLocations: " + veryPartialLocations);
        } else {
            PartialLocation pl =
                (PartialLocation) cloneInterMineObject(endLocation, PartialLocation.class);

            pl.setStartIsPartial(Boolean.TRUE);
            pl.setEndIsPartial(Boolean.FALSE);
            pl.setSubjectStart(new Integer(subjectLengthSoFar + 1));
            int newEndPos = pl.getSubjectStart().intValue()
                + (pl.getEnd().intValue() - pl.getStart().intValue());
            pl.setSubjectEnd(new Integer(newEndPos));
            osw.store(pl);
        }

    }


    // TODO 1) support pos/neg strand
    //      2) store chromosomes in id map -> avoid getObjectById
    //      3) map from chromosome to children -> less overlap comparasons
    //      4) check if a Location already exists?
    //      5) evidence collection for locations

    /**
     * Create new Location objects where required:
     *
     *  | = Location that should exist   ( = Location to create
     *
     * Chromosome
     *   | | ( (
     *   ChromosomeBand
     *     ( ( (
     *     Supercontig
     *       | (
     *       Contig
     *         |
     *         *features*
     *
     * @throws Exception if anything goes wrong
     */
    public void createLocations() throws Exception {
        osw.beginTransaction();

        // 0. Hold Chromosomes in map by id
        chrById = makeChromosomeMap();

        // 1. Find and hold locations of ChromosomeBands on Chromsomes
        makeChromosomeBandLocations();

        // 2. Find and hold locations of Supercontigs on Chromosomes
        //    Create locations of Supercontigs on ChromosomeBands
        makeSupercontigLocations();

        // 3. hold offsets of Contigs on Supercontigs
        //    create locations Contig->ChromosomeBand, Contig->Chromosome
        makeContigLocations();

        // 4. For all BioEntities located on Contigs compute other offsets on all parents
        Results results =
            PostProcessUtil.findLocationAndObjects(os, Contig.class, BioEntity.class, false);
        results.setBatchSize(500);

        Iterator resIter = results.iterator();

        // create map ChromsomeBands to avoid calling getObjectById
        // need to keep running query after each commit transaction
        Map idBands = new HashMap();
        Iterator bandIter = PostProcessUtil.selectObjectsOfClass(os, ChromosomeBand.class);
        while (bandIter.hasNext()) {
            ChromosomeBand band = (ChromosomeBand) bandIter.next();
            idBands.put(band.getId(), band);
        }
        LOG.info("built ChromosomeBand id map, size = " + idBands.keySet().size());

        // create map of Supercontigs to avoid calling getObjectById
        // need to keep running query after each commit transaction
        Map idScs = new HashMap();
        Iterator scIter = PostProcessUtil.selectObjectsOfClass(os, Supercontig.class);
        while (scIter.hasNext()) {
            Supercontig sc = (Supercontig) scIter.next();
            idScs.put(sc.getId(), sc);
        }
        LOG.info("built Supercontig id map, size = " + idScs.keySet().size());

        // maps from BioEntity to Location
        Map partialsOnChromosomes = new HashMap();
        Map partialsOnSupercontigs = new HashMap();
        Map partialsOnChromosomeBands = new HashMap();

        int i = 0, j = 0, k = 0;
        long start = System.currentTimeMillis();
        while (resIter.hasNext()) {
            i++;
            ResultsRow rr = (ResultsRow) resIter.next();
            Location locBioOnContig = (Location) rr.get(2);
            Integer contigId = (Integer) rr.get(0);
            BioEntity bio = (BioEntity) rr.get(1);
            SimpleLoc bioOnContig = new SimpleLoc(contigId.intValue(),
                                                  bio.getId().intValue(),
                                                  locBioOnContig);

            // first create location of feature on Chromosome
            SimpleLoc contigOnChr = (SimpleLoc) contigToChr.get(contigId);

            Chromosome chr = (Chromosome)
                chrById.get(new Integer(contigOnChr.getParentId()));
            Location bioOnChrLoc =
                createTransformedLocation(contigOnChr, bioOnContig, chr, bio);

            if (locBioOnContig instanceof PartialLocation) {
                addToMapOfLists(partialsOnChromosomes, bio, bioOnChrLoc);
            } else {
                osw.store(bioOnChrLoc);
            }

            SimpleLoc bioOnChr = new SimpleLoc(chr.getId().intValue(),
                                               bio.getId().intValue(),
                                               bioOnChrLoc);

            // create location of feature on Supercontig
            Set scs = (Set) chrToSc.get(chr.getId());
            if (scs != null) {
                Iterator iter = scs.iterator();
                while (iter.hasNext()) {
                    SimpleLoc scOnChr = (SimpleLoc) iter.next();
                    if (overlap(scOnChr, bioOnChr)) {
                        Supercontig sc = (Supercontig) idScs.get(new Integer(scOnChr.getChildId()));
                        Location bioOnScLoc = createLocation(sc, scOnChr, bio, bioOnChr);

                        if (bioOnScLoc instanceof PartialLocation) {
                            addToMapOfLists(partialsOnSupercontigs, bio, bioOnScLoc);
                        } else {
                            osw.store(bioOnScLoc);
                        }
                        j++;
                    }
                }
            }

            // create location of feature on ChromosomeBand
            Set bands = (Set) chrToBand.get(chr.getId());
            if (bands != null) {
                Iterator iter = bands.iterator();
                while (iter.hasNext()) {
                    SimpleLoc bandOnChr = (SimpleLoc) iter.next();
                    if (overlap(bandOnChr, bioOnChr)) {
                        ChromosomeBand band = (ChromosomeBand)
                            idBands.get(new Integer(bandOnChr.getChildId()));
                        Location bioOnBandLoc = createLocation(band, bandOnChr, bio, bioOnChr);

                        if (bioOnBandLoc instanceof PartialLocation) {
                            addToMapOfLists(partialsOnChromosomeBands, bio, bioOnBandLoc);
                        } else {
                            osw.store(bioOnBandLoc);
                        }
                        k++;
                    }
                }
            }
            if (i % 100 == 0) {
                long now = System.currentTimeMillis();
                LOG.info("Created " + i + " Chromosome, "
                         + j + " SuperContig locations and "
                         + k + " ChromosomeBand locations (avg = "
                         + ((60000L * i) / (now - start)) + " per minute)");
            }
        }

        chrById = null;
        bandToChr = null;
        chrToBand = null;
        chrToSc = null;
        scToChr = null;
        contigToSc = null;
        contigToChr = null;

        // process partials Locations
        processPartials(partialsOnChromosomes);
        processPartials(partialsOnSupercontigs);
        processPartials(partialsOnChromosomeBands);

        osw.commitTransaction();

        LOG.info("Stored " + i + " Locations between features and Chromosome.");
        LOG.info("Stored " + j + " Locations between features and Supercontig.");
        LOG.info("Stored " + k + " Locations between features and ChromosomeBand.");
    }

    /**
     * Create OverlapRelation objects for all overlapping LocatedSequenceFeatures by querying
     * objects that are located on chromosomes and overlap.
     * @param classNamesToIgnore a comma separated list of the names of those classes that should be
     * ignored when searching for overlaps.  Sub classes to these classes are ignored too
     * @param ignoreSelfMatches if true, don't create OverlapRelations between two objects of the
     * same class.
     * @throws Exception if anything goes wrong
     */
    public void createOverlapRelations(List classNamesToIgnore, boolean ignoreSelfMatches)
         throws Exception {
        osw.beginTransaction();
        Map chromosomeMap = makeChromosomeMap();
        Iterator chromosomeIdIter = chromosomeMap.keySet().iterator();
        while (chromosomeIdIter.hasNext()) {
            Integer id = (Integer) chromosomeIdIter.next();
            createSubjectOverlapRelations((Chromosome) chromosomeMap.get(id), classNamesToIgnore,
                                          ignoreSelfMatches);
        }
        osw.commitTransaction();
    }

    /**
     * Create OverlapRelation objects for locations that have the given subject.
     * @param classNamesToIgnore a comma separated list of the names of those classes that should be
     * ignored when searching for overlaps.  Sub classes to these classes are ignored too
     * @param ignoreSelfMatches if true, don't create OverlapRelations between two objects of the
     * same class.
     */
    private void createSubjectOverlapRelations(Chromosome subject, List classNamesToIgnore,
                                               boolean ignoreSelfMatches)
        throws Exception {
        LOG.info("Creating overlaps for " + subject + ", identifier: "
                 + subject.getIdentifier());

        Iterator overlapIterator =
            OverlapUtil.findOverlaps(os, subject, classNamesToIgnore, ignoreSelfMatches);

        int count = 0;

        while (overlapIterator.hasNext()) {
            Location overlapLocations[] = (Location[]) overlapIterator.next();

            Location location1 = overlapLocations[0];
            Location location2 = overlapLocations[1];

            InterMineObject lsf1 = location1.proxGetSubject();
            InterMineObject lsf2 = location2.proxGetSubject();

            OverlapRelation overlapRelation = (OverlapRelation)
                DynamicUtil.createObject(Collections.singleton(OverlapRelation.class));
            Set bioEntityCollection = overlapRelation.getBioEntities();
            bioEntityCollection.add(lsf1);
            bioEntityCollection.add(lsf2);

            ++count;

            osw.store(overlapRelation);
        }

        LOG.info("Stored " + count + " overlaps for " + subject + ", identifier: "
                 + subject.getIdentifier());
    }

    /**
     * Put key and value in the given map.  The values of the map are List, to which the new value
     * is appended.  The Lists are created if missing.
     */
    private void addToMapOfLists(Map map, Object key, Object value) {
        if (map.get(key) != null) {
{'fraction_non_alphanumeric': 0.09413630011824142, 'fraction_numerical': 0.0042459421691927336, 'mean_word_length': 1.9002416023692619, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7153512', 'n_tokens_mistral': 9860, 'n_tokens_neox': 9271, 'n_words': 2550}