<commit_before>/*
 * The Gemma project
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.loader.expression.geo;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import ubic.basecode.io.ByteArrayConverter;
import ubic.gemma.loader.expression.arrayDesign.ArrayDesignSequenceProcessingService;
import ubic.gemma.loader.expression.geo.model.GeoChannel;
import ubic.gemma.loader.expression.geo.model.GeoContact;
import ubic.gemma.loader.expression.geo.model.GeoData;
import ubic.gemma.loader.expression.geo.model.GeoDataset;
import ubic.gemma.loader.expression.geo.model.GeoPlatform;
import ubic.gemma.loader.expression.geo.model.GeoReplication;
import ubic.gemma.loader.expression.geo.model.GeoSample;
import ubic.gemma.loader.expression.geo.model.GeoSeries;
import ubic.gemma.loader.expression.geo.model.GeoSubset;
import ubic.gemma.loader.expression.geo.model.GeoValues;
import ubic.gemma.loader.expression.geo.model.GeoVariable;
import ubic.gemma.loader.expression.geo.model.GeoDataset.ExperimentType;
import ubic.gemma.loader.expression.geo.model.GeoDataset.PlatformType;
import ubic.gemma.loader.expression.geo.model.GeoReplication.ReplicationType;
import ubic.gemma.loader.expression.geo.model.GeoVariable.VariableType;
import ubic.gemma.loader.expression.geo.util.GeoConstants;
import ubic.gemma.loader.util.converter.Converter;
import ubic.gemma.loader.util.parser.ExternalDatabaseUtils;
import ubic.gemma.model.common.auditAndSecurity.Contact;
import ubic.gemma.model.common.auditAndSecurity.Person;
import ubic.gemma.model.common.description.BibliographicReference;
import ubic.gemma.model.common.description.Characteristic;
import ubic.gemma.model.common.description.DatabaseEntry;
import ubic.gemma.model.common.description.DatabaseType;
import ubic.gemma.model.common.description.ExternalDatabase;
import ubic.gemma.model.common.description.ExternalDatabaseService;
import ubic.gemma.model.common.description.LocalFile;
import ubic.gemma.model.common.description.OntologyEntry;
import ubic.gemma.model.common.quantitationtype.PrimitiveType;
import ubic.gemma.model.common.quantitationtype.QuantitationType;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.model.expression.arrayDesign.TechnologyType;
import ubic.gemma.model.expression.bioAssay.BioAssay;
import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.biomaterial.Treatment;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.expression.experiment.ExperimentalDesign;
import ubic.gemma.model.expression.experiment.ExperimentalFactor;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.model.expression.experiment.ExpressionExperimentSubSet;
import ubic.gemma.model.expression.experiment.FactorValue;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.model.genome.TaxonService;
import ubic.gemma.model.genome.biosequence.BioSequence;
import ubic.gemma.model.genome.biosequence.PolymerType;
import ubic.gemma.model.genome.biosequence.SequenceType;

/**
 * Convert GEO domain objects into Gemma objects. Usually we trigger this by passing in GeoDataset objects.
 * <p>
 * GEO has four basic kinds of objects: Platforms (ArrayDesigns), Samples (BioAssays), Series (Experiments) and DataSets
 * (which are curated Experiments). Note that a sample can belong to more than one series. A series can include more
 * than one dataset. See http://www.ncbi.nlm.nih.gov/projects/geo/info/soft2.html.
 * <p>
 * For our purposes, a usable expression data set is at first represented by a GEO "GDS" number (a curated dataset),
 * which corresponds to a series. HOWEVER, multiple datasets may go together to form a series (GSE). This can happen
 * when the "A" and "B" arrays were both run on the same samples. Thus we actually normally go by GSE.
 * <p>
 * This service can be used in database-aware or unaware states.
 * 
 * @author keshav
 * @author pavlidis
 * @version $Id$
 * @spring.bean id="geoConverter"
 * @spring.property name="externalDatabaseService" ref="externalDatabaseService"
 * @spring.property name="taxonService" ref="taxonService"
 */
public class GeoConverter implements Converter {

    /**
     * How often we tell the user about data processing (items per update)
     */
    private static final int LOGGING_VECTOR_COUNT_UPDATE = 2000;

    private static Log log = LogFactory.getLog( ArrayDesignSequenceProcessingService.class.getName() );

    /**
     * Initial guess at how many designelementdatavectors to allocate space for.
     */
    private static final int INITIAL_VECTOR_CAPACITY = 10000;

    private ExternalDatabaseService externalDatabaseService;

    private TaxonService taxonService;

    private ByteArrayConverter byteArrayConverter = new ByteArrayConverter();

    private ExternalDatabase geoDatabase;

    private Map<String, Map<String, CompositeSequence>> platformDesignElementMap = new HashMap<String, Map<String, CompositeSequence>>();

    private Map<String, Taxon> platformTaxonMap = new HashMap<String, Taxon>();

    private Collection<Object> results = new HashSet<Object>();

    private Map<String, ArrayDesign> seenPlatforms = new HashMap<String, ArrayDesign>();

    private ExternalDatabase genbank;

    /**
     * The scientific name used for rat species. FIXME this should be updated elsewhere; avoid this hardcoding.
     */
    private static final String RAT = "Rattus norvegicus";

    private static Map<String, String> organismDatabases = new HashMap<String, String>();

    static {
        organismDatabases.put( "Saccharomyces cerevisiae", "SGD" );
        organismDatabases.put( "Schizosaccharomyces pombe", "GeneDB" );
    }

    /**
     * Find the bioassay in the experiment that matches the one given.
     */
    @SuppressWarnings("unchecked")
    private boolean addMatchingBioAssayToSubSet( ExpressionExperimentSubSet subSet, String accession,
            ExpressionExperiment expExp ) {
        log.debug( "Seeking subset match for " + accession );
        Collection<BioAssay> experimentBioAssays = expExp.getBioAssays();
        for ( BioAssay assay : experimentBioAssays ) {
            String testAccession = assay.getAccession().getAccession();
            if ( testAccession.equals( accession ) ) {
                subSet.getBioAssays().add( assay );
                return true;
            }
        }
        return false;
    }

    /**
     * Remove old results. Call this prior to starting conversion of a full dataset.
     */
    public void clear() {
        results = new HashSet<Object>();
        seenPlatforms = new HashMap<String, ArrayDesign>();
        platformDesignElementMap = new HashMap<String, Map<String, CompositeSequence>>();
    }

    /**
     * @param seriesMap
     */
    @SuppressWarnings("unchecked")
    public Collection<Object> convert( Collection geoObjects ) {
        for ( Object geoObject : geoObjects ) {
            Object convertedObject = convert( geoObject );
            if ( convertedObject != null ) {
                if ( convertedObject instanceof Collection ) {
                    results.addAll( ( Collection ) convertedObject );
                } else {
                    results.add( convertedObject );
                }
            }
        }

        log.info( "Converted object tally:\n" + this );

        // log.debug( "Detailed object tree:" );
        // log.debug( PrettyPrinter.print( results ) );

        return results;
    }

    /**
     * @param geoObject
     */
    public Object convert( Object geoObject ) {
        if ( geoObject == null ) {
            log.warn( "Null object" );
            return null;
        }
        if ( geoObject instanceof Collection ) {
            return convert( ( Collection ) geoObject );
        } else if ( geoObject instanceof GeoDataset ) {
            return convertDataset( ( GeoDataset ) geoObject );
        } else if ( geoObject instanceof GeoSeries ) { // typically we start here, with a series.
            return convertSeries( ( GeoSeries ) geoObject );
        } else if ( geoObject instanceof GeoSubset ) {
            throw new IllegalArgumentException( "Can't deal with " + geoObject.getClass().getName() + " ('" + geoObject
                    + "')" );
        } else if ( geoObject instanceof GeoSample ) {
            throw new IllegalArgumentException( "Can't deal with " + geoObject.getClass().getName() + " ('" + geoObject
                    + "')" );
        } else if ( geoObject instanceof GeoPlatform ) {
            return convertPlatform( ( GeoPlatform ) geoObject );
        } else {
            throw new IllegalArgumentException( "Can't deal with " + geoObject.getClass().getName() + " ('" + geoObject
                    + "')" );
        }

    }

    /**
     * GEO does not keep track of 'biomaterials' that make up different channels. Therefore the two channels effectively
     * make up a single biomaterial, as far as we're concerned. We're losing information here.
     * 
     * @param sample
     * @param channel
     * @return
     */
    @SuppressWarnings("unchecked")
    private BioMaterial convertChannel( GeoSample sample, GeoChannel channel, BioMaterial bioMaterial ) {
        if ( bioMaterial == null ) return null;
        log.debug( "Sample: " + sample.getGeoAccession() + " - Converting channel " + channel.getSourceName() );

        bioMaterial.setDescription( ( bioMaterial.getDescription() == null ? "" : bioMaterial.getDescription() + ";" )
                + "Channel " + channel.getChannelNumber() );

        if ( !StringUtils.isBlank( channel.getGrowthProtocol() ) ) {
            Treatment treatment = Treatment.Factory.newInstance();
            treatment.setName( sample.getGeoAccession() + " channel " + channel.getChannelNumber() + " treatment" );
            treatment.setDescription( channel.getGrowthProtocol() );
            bioMaterial.getTreatments().add( treatment );
        }

        if ( !StringUtils.isBlank( channel.getTreatmentProtocol() ) ) {
            Treatment treatment = Treatment.Factory.newInstance();
            treatment.setName( sample.getGeoAccession() + " channel " + channel.getChannelNumber() + " growth" );
            treatment.setDescription( channel.getTreatmentProtocol() );
            bioMaterial.getTreatments().add( treatment );
        }

        if ( !StringUtils.isBlank( channel.getExtractProtocol() ) ) {
            Treatment treatment = Treatment.Factory.newInstance();
            treatment.setName( sample.getGeoAccession() + " channel " + channel.getChannelNumber() + " extraction" );
            treatment.setDescription( channel.getExtractProtocol() );
            bioMaterial.getTreatments().add( treatment );
        }

        if ( !StringUtils.isBlank( channel.getLabelProtocol() ) ) {
            Treatment treatment = Treatment.Factory.newInstance();
            treatment.setName( sample.getGeoAccession() + " channel " + channel.getChannelNumber() + " label" );
            treatment.setDescription( channel.getLabelProtocol() );
            bioMaterial.getTreatments().add( treatment );
        }

        for ( String characteristic : channel.getCharacteristics() ) {
            Characteristic gemmaChar = Characteristic.Factory.newInstance();

            characteristic = trimString( characteristic );

            gemmaChar.setCategory( "GEO Sample characteristic" );
            gemmaChar.setValue( characteristic );
            bioMaterial.getCharacteristics().add( gemmaChar );
        }

        if ( channel.getSourceName() != null ) {
            Characteristic sourceChar = Characteristic.Factory.newInstance();
            sourceChar.setCategory( "GEO Sample source" );
            String characteristic = trimString( channel.getSourceName() );
            sourceChar.setValue( characteristic );
            bioMaterial.getCharacteristics().add( sourceChar );
        }

        if ( channel.getOrganism() != null ) {
            Taxon taxon = Taxon.Factory.newInstance();
            taxon.setScientificName( channel.getOrganism() );
            bioMaterial.setSourceTaxon( taxon );
        }

        if ( channel.getMolecule() != null ) {
            Characteristic moleculeChar = Characteristic.Factory.newInstance();
            moleculeChar.setCategory( "GEO Sample molecule" );
            String characteristic = trimString( channel.getMolecule().toString() );
            moleculeChar.setValue( characteristic );
            bioMaterial.getCharacteristics().add( moleculeChar );
        }

        if ( channel.getLabel() != null ) {
            Characteristic labelChar = Characteristic.Factory.newInstance();
            labelChar.setCategory( "GEO Sample label" );
            String characteristic = trimString( channel.getLabel() );
            labelChar.setValue( characteristic );
            bioMaterial.getCharacteristics().add( labelChar );
        }

        return bioMaterial;
    }

    private String trimString( String characteristic ) {
        if ( characteristic.length() > 255 ) {
            log.warn( "** Characteristic too long: " + characteristic + " - will truncate - ****" );
            characteristic = characteristic.substring( 0, 199 ) + " (truncated at 200 characters)";
        }
        return characteristic;
    }

    /**
     * @param contact
     * @return
     */
    private Person convertContact( GeoContact contact ) {
        Person result = Person.Factory.newInstance();
        result.setAddress( contact.getCity() + " " + contact.getState() + " " + contact.getCountry() + " "
                + contact.getPostCode() );
        result.setPhone( contact.getPhone() );
        result.setName( contact.getName() );
        result.setEmail( contact.getEmail() );
        result.setFax( contact.getFax() );
        result.setURL( contact.getWebLink() );

        return result;
    }

    /**
     * Take contact and contributer information from a GeoSeries and put it in the ExpressionExperiment.
     * 
     * @param series
     * @param expExp
     */
    @SuppressWarnings("unchecked")
    private void convertContacts( GeoSeries series, ExpressionExperiment expExp ) {
        expExp.getInvestigators().add( convertContact( series.getContact() ) );
        if ( series.getContributers().size() > 0 ) {
            expExp.setDescription( expExp.getDescription() + "\nContributers: " );
            for ( GeoContact contributer : series.getContributers() ) {
                expExp.setDescription( expExp.getDescription() + " " + contributer.getName() );
                expExp.getInvestigators().add( convertContact( contributer ) );
            }
            expExp.setDescription( expExp.getDescription() + "\n" );
        }
    }

    /**
     * Convert a vector of strings into a byte[] for saving in the database. . Blanks(missing values) are treated as NAN
     * (double), 0 (integer), false (booleans) or just empty strings (strings). Other invalid values are treated the
     * same way as missing data (to keep the parser from failing when dealing with strange GEO files that have values
     * like "Error" for an expression value).
     * 
     * @param vector of Strings to be converted to primitive values (double, int etc)
     * @param qt The quantitation type for the values to be converted.
     * @return
     */
    protected byte[] convertData( List<Object> vector, QuantitationType qt ) {

        if ( vector == null || vector.size() == 0 ) return null;

        boolean containsAtLeastOneNonNull = false;
        for ( Object string : vector ) {
            if ( string != null ) {
                containsAtLeastOneNonNull = true;
                break;
            }
        }

        if ( !containsAtLeastOneNonNull ) {
            if ( log.isDebugEnabled() ) {
                log.debug( "No data for " + qt + " in vector of length " + vector.size() );
            }
            return null;
        }

        List<Object> toConvert = new ArrayList<Object>();
        PrimitiveType pt = qt.getRepresentation();
        int numMissing = 0;
        for ( Object rawValue : vector ) {
            if ( rawValue == null ) {
                numMissing++;
                handleMissing( toConvert, pt );
            } else if ( rawValue instanceof String ) { // needs to be coverted.
                try {
                    if ( pt.equals( PrimitiveType.DOUBLE ) ) {
                        toConvert.add( Double.parseDouble( ( String ) rawValue ) );
                    } else if ( pt.equals( PrimitiveType.STRING ) || pt.equals( PrimitiveType.STRINGARRAY ) ) {
                        // FIXME BUG 741 - converter makes it a tabbed string, which is a no-no.
                        toConvert.add( ( String ) rawValue );
                    } else if ( pt.equals( PrimitiveType.CHAR ) ) {
                        if ( ( ( String ) rawValue ).length() != 1 ) {
                            throw new IllegalStateException( "Attempt to cast a string of length "
                                    + ( ( String ) rawValue ).length() + " to a char" );
                        }
                        toConvert.add( ( Character ) ( ( String ) rawValue ).toCharArray()[0] );
                    } else if ( pt.equals( PrimitiveType.INT ) ) {
                        toConvert.add( Integer.parseInt( ( String ) rawValue ) );
                    } else if ( pt.equals( PrimitiveType.BOOLEAN ) ) {
                        toConvert.add( Boolean.parseBoolean( ( String ) rawValue ) );

                    } else {
                        throw new UnsupportedOperationException( "Data vectors of type " + pt + " not supported" );
                    }
                } catch ( NumberFormatException e ) {
                    numMissing++;
                    handleMissing( toConvert, pt );
                } catch ( NullPointerException e ) {
                    numMissing++;
                    handleMissing( toConvert, pt );
                }
            } else { // use as is.
                toConvert.add( rawValue );
            }
        }

        if ( numMissing == vector.size() ) {
            return null;
        }

        return byteArrayConverter.toBytes( toConvert.toArray() );
    }

    /**
     * Often-needed generation of a valid databaseentry object.
     * 
     * @param geoData
     * @return
     */
    private DatabaseEntry convertDatabaseEntry( GeoData geoData ) {
        DatabaseEntry result = DatabaseEntry.Factory.newInstance();

        initGeoExternalDatabase();

        result.setExternalDatabase( this.geoDatabase );
        result.setAccession( geoData.getGeoAccession() );

        return result;
    }

    /**
     * @param geoDataset
     */
    private ExpressionExperiment convertDataset( GeoDataset geoDataset ) {

        if ( geoDataset.getSeries().size() == 0 ) {
            throw new IllegalArgumentException( "GEO Dataset must have associated series" );
        }

        if ( geoDataset.getSeries().size() > 1 ) {
            throw new UnsupportedOperationException( "GEO Dataset can only be associated with one series" );
        }

        Collection<ExpressionExperiment> results = this.convertSeries( geoDataset.getSeries().iterator().next() );
        assert results.size() == 1; // unless we have multiple species, not possible.
        return results.iterator().next();
    }

    /**
     * @param dataset
     * @param expExp
     */
    private ExpressionExperiment convertDataset( GeoDataset geoDataset, ExpressionExperiment expExp ) {

        /*
         * First figure out of there are any samples for this data set. It could be that they were duplicates of ones
         * found in other series, so were skipped. See GeoDatasetService
         */
        if ( this.getDatasetSamples( geoDataset ).size() == 0 ) {
            log.info( "No samples remain for " + geoDataset + ", nothing to do" );
            return expExp;
        }

        log.info( "Converting dataset:" + geoDataset );

        convertDatasetDescriptions( geoDataset, expExp );

        GeoPlatform platform = geoDataset.getPlatform();
        ArrayDesign ad = seenPlatforms.get( platform.getGeoAccession() );
        if ( ad == null ) {
            throw new IllegalStateException( "ArrayDesigns must be converted before datasets - didn't find "
                    + geoDataset.getPlatform() );
        }

        ad.setDescription( ad.getDescription() + "\nFrom " + platform.getGeoAccession() + "\nLast Updated: "
                + platform.getLastUpdateDate() );

        LocalFile arrayDesignRawFile = convertSupplementaryFileToLocalFile( platform );
        if ( arrayDesignRawFile != null ) {
            Collection<LocalFile> arrayDesignLocalFiles = ad.getLocalFiles();
            if ( arrayDesignLocalFiles == null ) {
                arrayDesignLocalFiles = new HashSet<LocalFile>();
            }
            arrayDesignLocalFiles.add( arrayDesignRawFile );
            ad.setLocalFiles( arrayDesignLocalFiles );
        }

        convertDataSetDataVectors( geoDataset.getSeries().iterator().next().getValues(), geoDataset, expExp );

        convertSubsetAssociations( expExp, geoDataset );
        return expExp;

    }

    /**
     * Use this when we don't have a GDS for a GSE.
     * 
     * @param geoSeries
     * @param expExp
     */
    private void convertSeriesDataVectors( GeoSeries geoSeries, ExpressionExperiment expExp ) {
        /*
         * Tricky thing is that series contains data from multiple platforms.
         */
        DatasetCombiner datasetCombiner = new DatasetCombiner();
        Map<GeoPlatform, List<GeoSample>> platformSamples = datasetCombiner.getPlatformSampleMap( geoSeries );

        for ( GeoPlatform platform : platformSamples.keySet() ) {
            List<GeoSample> samples = platformSamples.get( platform );
            log.info( samples.size() + " samples on " + platform );
            convertVectorsForPlatform( geoSeries.getValues(), expExp, samples, platform );
            geoSeries.getValues().clear( platform );
        }

    }

    /**
     * Convert the GEO data into DesignElementDataVectors associated with the ExpressionExperiment
     * 
     * @param geoDataset Source of the data
     * @param expExp ExpressionExperiment to fill in.
     */
    private void convertDataSetDataVectors( GeoValues values, GeoDataset geoDataset, ExpressionExperiment expExp ) {
        List<GeoSample> datasetSamples = new ArrayList<GeoSample>( getDatasetSamples( geoDataset ) );
        log.info( datasetSamples.size() + " samples in " + geoDataset );
        GeoPlatform geoPlatform = geoDataset.getPlatform();

        convertVectorsForPlatform( values, expExp, datasetSamples, geoPlatform );

        values.clear( geoPlatform );
    }

    /**
     * @param expExp
     * @param datasetSamples
     * @param geoPlatform
     */
    private void convertVectorsForPlatform( GeoValues values, ExpressionExperiment expExp,
            List<GeoSample> datasetSamples, GeoPlatform geoPlatform ) {
        assert datasetSamples.size() > 0 : "No samples in dataset";

        log.info( "Converting vectors for " + geoPlatform.getGeoAccession() + ", " + datasetSamples.size()
                + " samples." );

        BioAssayDimension bioAssayDimension = convertGeoSampleList( datasetSamples, expExp );

        if ( bioAssayDimension.getBioAssays().size() == 0 )
            throw new IllegalStateException( "No bioAssays in the BioAssayDimension" );

        sanityCheckQuantitationTypes( datasetSamples );

        List<String> quantitationTypes = datasetSamples.iterator().next().getColumnNames();
        List<String> quantitationTypeDescriptions = datasetSamples.iterator().next().getColumnDescriptions();
        boolean first = true;

        /*
         * For the data that are put in 'datasets' (GDS), we know the type of data, but it can be misleading (e.g., Affy
         * data is 'counts'). For others we just have free text in the column descriptions
         */

        int quantitationTypeIndex = 0;
        for ( String quantitationType : quantitationTypes ) {

            // skip the first quantitationType, it's the ID or ID_REF.
            if ( first ) {
                first = false;
                // quantitationTypeIndex++; // don't count the first column, don't increment.
                continue;
            }

            log.info( "Processing " + quantitationType + " (column=" + quantitationTypeIndex + ")" );

            Map<String, List<Object>> dataVectors = makeDataVectors( values, datasetSamples, quantitationTypeIndex );

            if ( dataVectors == null ) {
                log.info( "No data for " + quantitationType + " (column=" + quantitationTypeIndex + ")" );
                quantitationTypeIndex++;
                continue;
            } else {
                log.info( "Got " + dataVectors.size() + " data vectors for " + quantitationType + " (column="
                        + quantitationTypeIndex + ")" );
            }

            QuantitationType qt = QuantitationType.Factory.newInstance();
            qt.setName( quantitationType );
            String description = quantitationTypeDescriptions.get( quantitationTypeIndex + 1 );
            qt.setDescription( description );
            QuantitationTypeParameterGuesser.guessQuantitationTypeParameters( qt, quantitationType, description );

            expExp.getQuantitationTypes().add( qt );

            int count = 0;
            for ( String designElementName : dataVectors.keySet() ) {
                List<Object> dataVector = dataVectors.get( designElementName );
                if ( dataVector == null || dataVector.size() == 0 ) continue;

                DesignElementDataVector vector = convertDesignElementDataVector( geoPlatform, expExp,
                        bioAssayDimension, designElementName, dataVector, qt );

                if ( vector == null ) {
                    if ( log.isDebugEnabled() )
                        log.debug( "Null vector for DE=" + designElementName + " QT=" + quantitationType );
                    continue;
                }

                if ( log.isTraceEnabled() ) {
                    log.trace( designElementName + " " + qt.getName() + " " + qt.getRepresentation() + " "
                            + dataVector.size() + " elements in vector" );
                }

                expExp.getDesignElementDataVectors().add( vector );

                if ( log.isDebugEnabled() && ++count % LOGGING_VECTOR_COUNT_UPDATE == 0 ) {
                    log.debug( count + " Data vectors added" );
                }
            }
            if ( log.isDebugEnabled() ) {
                log.debug( count + " Data vectors added for '" + quantitationType + "'" );
            }
            quantitationTypeIndex++;
        }
        log.info( "Total of " + expExp.getDesignElementDataVectors().size() + " vectors so far..." );
    }

    /**
     * @param geoDataset
     * @param expExp
     */
    private void convertDatasetDescriptions( GeoDataset geoDataset, ExpressionExperiment expExp ) {
        if ( StringUtils.isEmpty( expExp.getDescription() ) ) {
            expExp.setDescription( geoDataset.getDescription() ); // probably not empty.
        }

        expExp.setDescription( expExp.getDescription() + "\nIncludes " + geoDataset.getGeoAccession() + ".\n" );
        if ( StringUtils.isNotEmpty( geoDataset.getUpdateDate() ) ) {
            expExp.setDescription( expExp.getDescription() + " Update date: " + geoDataset.getUpdateDate() + ".\n" );
        }

        if ( StringUtils.isEmpty( expExp.getName() ) ) {
            expExp.setName( geoDataset.getTitle() );
        } else {
            expExp.setDescription( expExp.getDescription() + " Dataset description " + geoDataset.getGeoAccession()
                    + ": " + geoDataset.getTitle() + ".\n" );
        }
    }

    /**
     * @param geoDataset
     * @param expExp
     * @param bioAssayDimension
     * @param designElementName
     * @param dataVector to convert.
     * @return vector, or null if the dataVector was null or empty.
     */
    private DesignElementDataVector convertDesignElementDataVector( GeoPlatform geoPlatform,
            ExpressionExperiment expExp, BioAssayDimension bioAssayDimension, String designElementName,
            List<Object> dataVector, QuantitationType qt ) {

        if ( dataVector == null || dataVector.size() == 0 ) return null;

        byte[] blob = convertData( dataVector, qt );
        if ( blob == null ) { // all missing etc.
            if ( log.isDebugEnabled() ) log.debug( "All missing values for DE=" + designElementName + " QT=" + qt );
            return null;
        }
        if ( log.isDebugEnabled() ) {
            log.debug( blob.length + " bytes for " + dataVector.size() + " raw elements" );
        }

        CompositeSequence compositeSequence = platformDesignElementMap.get(
                convertPlatform( geoPlatform ).getShortName() ).get( designElementName );

        if ( compositeSequence == null ) {
            assert compositeSequence != null : "No composite sequence " + designElementName;
        }

        if ( compositeSequence.getBiologicalCharacteristic() != null
                && compositeSequence.getBiologicalCharacteristic().getSequenceDatabaseEntry() != null ) {
            assert compositeSequence.getBiologicalCharacteristic().getSequenceDatabaseEntry().getExternalDatabase()
                    .getName() != null;
        }

        DesignElementDataVector vector = DesignElementDataVector.Factory.newInstance();
        vector.setDesignElement( compositeSequence );
        vector.setExpressionExperiment( expExp );

        vector.setBioAssayDimension( bioAssayDimension );
        vector.setQuantitationType( qt );
        vector.setData( blob );
        return vector;
    }

    /**
     * @param datasetSamples List of GeoSamples to be matched up with BioAssays.
     * @param expExp ExpresssionExperiment
     * @return BioAssayDimension representing the samples.
     */
    @SuppressWarnings("unchecked")
    private BioAssayDimension convertGeoSampleList( List<GeoSample> datasetSamples, ExpressionExperiment expExp ) {
        BioAssayDimension resultBioAssayDimension = BioAssayDimension.Factory.newInstance();

        StringBuilder bioAssayDimName = new StringBuilder();
        Collections.sort( datasetSamples );
        for ( GeoSample sample : datasetSamples ) {
            boolean found = false;
            String sampleAcc = sample.getGeoAccession();
            bioAssayDimName.append( sampleAcc + "," ); // this is rather silly!
            found = matchSampleToBioAssay( expExp, resultBioAssayDimension, sampleAcc );
            if ( !found ) {
                // this is normal because not all headings are
                // sample ids.
                log.warn( "No bioassay match for " + sampleAcc );
            }
        }
        log.info( resultBioAssayDimension.getBioAssays().size() + " Bioassays in biodimension" );
        resultBioAssayDimension.setName( formatName( bioAssayDimName ) );
        resultBioAssayDimension.setDescription( bioAssayDimName.toString() );
        return resultBioAssayDimension;
    }

    /**
     * @param platform
     */
    @SuppressWarnings("unchecked")
    private ArrayDesign convertPlatform( GeoPlatform platform ) {

        if ( seenPlatforms.containsKey( platform.getGeoAccession() ) ) {
            return ( seenPlatforms.get( platform.getGeoAccession() ) );
        }

        if ( isSage( platform ) ) {
            throw new UnsupportedOperationException( "This data set uses SAGE, it cannot be handled yet" );
        }

        log.info( "Converting platform: " + platform.getGeoAccession() );
        ArrayDesign arrayDesign = createMinimalArrayDesign( platform );

        platformDesignElementMap.put( arrayDesign.getShortName(), new HashMap<String, CompositeSequence>() );

        Taxon taxon = convertPlatformOrganism( platform );

        // convert the design element information.
        String identifier = determinePlatformIdentifier( platform );
        Collection<String> externalReferences = determinePlatformExternalReferenceIdentifier( platform );
        String descriptionColumn = determinePlatformDescriptionColumn( platform );
        String sequenceColumn = determinePlatformSequenceColumn( platform );
        ExternalDatabase externalDb = determinePlatformExternalDatabase( platform );

        List<String> identifiers = platform.getColumnData( identifier );
        List<String> descriptions = platform.getColumnData( descriptionColumn );
        List<String> sequences = platform.getColumnData( sequenceColumn );
        List<String> cloneIdentifiers = platform.getColumnData( "CLONE_ID" );

        List<List<String>> externalRefs = null;
        if ( externalReferences != null ) {
            externalRefs = platform.getColumnData( externalReferences );
        }

        assert identifier != null;

        if ( externalRefs != null ) {
            assert externalRefs.iterator().next().size() == identifiers.size() : "Unequal numbers of identifiers and external references! "
                    + externalRefs.iterator().next().size() + " != " + identifiers.size();
        }
        assert cloneIdentifiers == null || cloneIdentifiers.size() == identifiers.size();

        if ( log.isDebugEnabled() ) {
            log.debug( "Converting " + identifiers.size() + " probe identifiers on GEO platform "
                    + platform.getGeoAccession() );
        }

        Iterator<String> descIter = null;

        if ( descriptions != null ) {
            descIter = descriptions.iterator();
        }

        // much faster than hashset to add to when it gets large. (because equals isn't implemented right... FIXME)
        Collection compositeSequences = new ArrayList( 5000 );
        int i = 0; // to get sequences, if we have them, and clone identifiers.
        for ( String id : identifiers ) {
            String externalAccession = null;
            if ( externalRefs != null ) {
                externalAccession = getExternalAccession( externalRefs, i );
            }

            String cloneIdentifier = cloneIdentifiers == null ? null : cloneIdentifiers.get( i );

            String description = "";
            if ( externalAccession != null ) {
                String[] refs = externalAccession.split( "," );
                if ( refs.length > 1 ) {
                    description = "Multiple external sequence references: " + externalAccession + "; ";
                    externalAccession = refs[0];
                }
            }

            if ( descIter != null ) description = description + " " + descIter.next();

            CompositeSequence cs = CompositeSequence.Factory.newInstance();
            cs.setName( id );
            cs.setDescription( description );
            cs.setArrayDesign( arrayDesign );

            BioSequence bs = createMinimalBioSequence( taxon );

            boolean isImage = false;
            if ( StringUtils.isNotBlank( cloneIdentifier ) ) {
                bs.setName( cloneIdentifier );
                isImage = cloneIdentifier.startsWith( "IMAGE" );
            }

            /*
             * If we are given a sequence, we don't need the genbank identifier, which is probably not correct anyway.
             */
            if ( sequences != null && StringUtils.isNotBlank( sequences.get( i ) ) ) {
                bs.setSequence( sequences.get( i ) );
                bs.setIsApproximateLength( false );
                bs.setLength( new Long( bs.getSequence().length() ) );
                bs.setType( SequenceType.DNA );
                bs.setName( id + "_sequence" );
                bs.setDescription( "Sequence provided by manufacturer, used in leiu of " + externalAccession );
            } else if ( externalAccession != null && !isImage && externalDb != null ) {
                /*
                 * We don't use this if we have an IMAGE clone because the accession might be wrong (e.g., for a
                 * Refseq). During persisting the IMAGE clone will be replaced with the 'real' thing.
                 */

                DatabaseEntry dbe = createDatabaseEntry( externalDb, externalAccession, bs );
                bs.setSequenceDatabaseEntry( dbe );
            }

            /*
             * If we have no basis for describing the sequence, we have to skip it.
             */
            if ( StringUtils.isBlank( externalAccession ) && StringUtils.isBlank( cloneIdentifier ) ) {
                if ( log.isDebugEnabled() ) {
                    log.debug( "Blank external reference and clone id for " + cs + " on " + arrayDesign
                            + ", no biological characteristic can be added." );
                }
            } else {
                cs.setBiologicalCharacteristic( bs );
            }

            compositeSequences.add( cs );

            platformDesignElementMap.get( arrayDesign.getShortName() ).put( id, cs );
            i++;
        }<commit_msg>bug 797: reject refsseq accessions as external sequence identifiers.<commit_after>/*
 * The Gemma project
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.loader.expression.geo;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import ubic.basecode.io.ByteArrayConverter;
import ubic.gemma.loader.expression.arrayDesign.ArrayDesignSequenceProcessingService;
import ubic.gemma.loader.expression.geo.model.GeoChannel;
import ubic.gemma.loader.expression.geo.model.GeoContact;
import ubic.gemma.loader.expression.geo.model.GeoData;
import ubic.gemma.loader.expression.geo.model.GeoDataset;
import ubic.gemma.loader.expression.geo.model.GeoPlatform;
import ubic.gemma.loader.expression.geo.model.GeoReplication;
import ubic.gemma.loader.expression.geo.model.GeoSample;
import ubic.gemma.loader.expression.geo.model.GeoSeries;
import ubic.gemma.loader.expression.geo.model.GeoSubset;
import ubic.gemma.loader.expression.geo.model.GeoValues;
import ubic.gemma.loader.expression.geo.model.GeoVariable;
import ubic.gemma.loader.expression.geo.model.GeoDataset.ExperimentType;
import ubic.gemma.loader.expression.geo.model.GeoDataset.PlatformType;
import ubic.gemma.loader.expression.geo.model.GeoReplication.ReplicationType;
import ubic.gemma.loader.expression.geo.model.GeoVariable.VariableType;
import ubic.gemma.loader.expression.geo.util.GeoConstants;
import ubic.gemma.loader.util.converter.Converter;
import ubic.gemma.loader.util.parser.ExternalDatabaseUtils;
import ubic.gemma.model.common.auditAndSecurity.Contact;
import ubic.gemma.model.common.auditAndSecurity.Person;
import ubic.gemma.model.common.description.BibliographicReference;
import ubic.gemma.model.common.description.Characteristic;
import ubic.gemma.model.common.description.DatabaseEntry;
import ubic.gemma.model.common.description.DatabaseType;
import ubic.gemma.model.common.description.ExternalDatabase;
import ubic.gemma.model.common.description.ExternalDatabaseService;
import ubic.gemma.model.common.description.LocalFile;
import ubic.gemma.model.common.description.OntologyEntry;
import ubic.gemma.model.common.quantitationtype.PrimitiveType;
import ubic.gemma.model.common.quantitationtype.QuantitationType;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.model.expression.arrayDesign.TechnologyType;
import ubic.gemma.model.expression.bioAssay.BioAssay;
import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.biomaterial.Treatment;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.expression.experiment.ExperimentalDesign;
import ubic.gemma.model.expression.experiment.ExperimentalFactor;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.model.expression.experiment.ExpressionExperimentSubSet;
import ubic.gemma.model.expression.experiment.FactorValue;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.model.genome.TaxonService;
import ubic.gemma.model.genome.biosequence.BioSequence;
import ubic.gemma.model.genome.biosequence.PolymerType;
import ubic.gemma.model.genome.biosequence.SequenceType;

/**
 * Convert GEO domain objects into Gemma objects. Usually we trigger this by passing in GeoDataset objects.
 * <p>
 * GEO has four basic kinds of objects: Platforms (ArrayDesigns), Samples (BioAssays), Series (Experiments) and DataSets
 * (which are curated Experiments). Note that a sample can belong to more than one series. A series can include more
 * than one dataset. See http://www.ncbi.nlm.nih.gov/projects/geo/info/soft2.html.
 * <p>
 * For our purposes, a usable expression data set is at first represented by a GEO "GDS" number (a curated dataset),
 * which corresponds to a series. HOWEVER, multiple datasets may go together to form a series (GSE). This can happen
 * when the "A" and "B" arrays were both run on the same samples. Thus we actually normally go by GSE.
 * <p>
 * This service can be used in database-aware or unaware states.
 * 
 * @author keshav
 * @author pavlidis
 * @version $Id$
 * @spring.bean id="geoConverter"
 * @spring.property name="externalDatabaseService" ref="externalDatabaseService"
 * @spring.property name="taxonService" ref="taxonService"
 */
public class GeoConverter implements Converter {

    /**
     * How often we tell the user about data processing (items per update)
     */
    private static final int LOGGING_VECTOR_COUNT_UPDATE = 2000;

    private static Log log = LogFactory.getLog( ArrayDesignSequenceProcessingService.class.getName() );

    /**
     * Initial guess at how many designelementdatavectors to allocate space for.
     */
    private static final int INITIAL_VECTOR_CAPACITY = 10000;

    private ExternalDatabaseService externalDatabaseService;

    private TaxonService taxonService;

    private ByteArrayConverter byteArrayConverter = new ByteArrayConverter();

    private ExternalDatabase geoDatabase;

    private Map<String, Map<String, CompositeSequence>> platformDesignElementMap = new HashMap<String, Map<String, CompositeSequence>>();

    private Map<String, Taxon> platformTaxonMap = new HashMap<String, Taxon>();

    private Collection<Object> results = new HashSet<Object>();

    private Map<String, ArrayDesign> seenPlatforms = new HashMap<String, ArrayDesign>();

    private ExternalDatabase genbank;

    /**
     * The scientific name used for rat species. FIXME this should be updated elsewhere; avoid this hardcoding.
     */
    private static final String RAT = "Rattus norvegicus";

    private static Map<String, String> organismDatabases = new HashMap<String, String>();

    static {
        organismDatabases.put( "Saccharomyces cerevisiae", "SGD" );
        organismDatabases.put( "Schizosaccharomyces pombe", "GeneDB" );
    }

    /**
     * Find the bioassay in the experiment that matches the one given.
     */
    @SuppressWarnings("unchecked")
    private boolean addMatchingBioAssayToSubSet( ExpressionExperimentSubSet subSet, String accession,
            ExpressionExperiment expExp ) {
        log.debug( "Seeking subset match for " + accession );
        Collection<BioAssay> experimentBioAssays = expExp.getBioAssays();
        for ( BioAssay assay : experimentBioAssays ) {
            String testAccession = assay.getAccession().getAccession();
            if ( testAccession.equals( accession ) ) {
                subSet.getBioAssays().add( assay );
                return true;
            }
        }
        return false;
    }

    /**
     * Remove old results. Call this prior to starting conversion of a full dataset.
     */
    public void clear() {
        results = new HashSet<Object>();
        seenPlatforms = new HashMap<String, ArrayDesign>();
        platformDesignElementMap = new HashMap<String, Map<String, CompositeSequence>>();
    }

    /**
     * @param seriesMap
     */
    @SuppressWarnings("unchecked")
    public Collection<Object> convert( Collection geoObjects ) {
        for ( Object geoObject : geoObjects ) {
            Object convertedObject = convert( geoObject );
            if ( convertedObject != null ) {
                if ( convertedObject instanceof Collection ) {
                    results.addAll( ( Collection ) convertedObject );
                } else {
                    results.add( convertedObject );
                }
            }
        }

        log.info( "Converted object tally:\n" + this );

        // log.debug( "Detailed object tree:" );
        // log.debug( PrettyPrinter.print( results ) );

        return results;
    }

    /**
     * @param geoObject
     */
    public Object convert( Object geoObject ) {
        if ( geoObject == null ) {
            log.warn( "Null object" );
            return null;
        }
        if ( geoObject instanceof Collection ) {
            return convert( ( Collection ) geoObject );
        } else if ( geoObject instanceof GeoDataset ) {
            return convertDataset( ( GeoDataset ) geoObject );
        } else if ( geoObject instanceof GeoSeries ) { // typically we start here, with a series.
            return convertSeries( ( GeoSeries ) geoObject );
        } else if ( geoObject instanceof GeoSubset ) {
            throw new IllegalArgumentException( "Can't deal with " + geoObject.getClass().getName() + " ('" + geoObject
                    + "')" );
        } else if ( geoObject instanceof GeoSample ) {
            throw new IllegalArgumentException( "Can't deal with " + geoObject.getClass().getName() + " ('" + geoObject
                    + "')" );
        } else if ( geoObject instanceof GeoPlatform ) {
            return convertPlatform( ( GeoPlatform ) geoObject );
        } else {
            throw new IllegalArgumentException( "Can't deal with " + geoObject.getClass().getName() + " ('" + geoObject
                    + "')" );
        }

    }

    /**
     * GEO does not keep track of 'biomaterials' that make up different channels. Therefore the two channels effectively
     * make up a single biomaterial, as far as we're concerned. We're losing information here.
     * 
     * @param sample
     * @param channel
     * @return
     */
    @SuppressWarnings("unchecked")
    private BioMaterial convertChannel( GeoSample sample, GeoChannel channel, BioMaterial bioMaterial ) {
        if ( bioMaterial == null ) return null;
        log.debug( "Sample: " + sample.getGeoAccession() + " - Converting channel " + channel.getSourceName() );

        bioMaterial.setDescription( ( bioMaterial.getDescription() == null ? "" : bioMaterial.getDescription() + ";" )
                + "Channel " + channel.getChannelNumber() );

        if ( !StringUtils.isBlank( channel.getGrowthProtocol() ) ) {
            Treatment treatment = Treatment.Factory.newInstance();
            treatment.setName( sample.getGeoAccession() + " channel " + channel.getChannelNumber() + " treatment" );
            treatment.setDescription( channel.getGrowthProtocol() );
            bioMaterial.getTreatments().add( treatment );
        }

        if ( !StringUtils.isBlank( channel.getTreatmentProtocol() ) ) {
            Treatment treatment = Treatment.Factory.newInstance();
            treatment.setName( sample.getGeoAccession() + " channel " + channel.getChannelNumber() + " growth" );
            treatment.setDescription( channel.getTreatmentProtocol() );
            bioMaterial.getTreatments().add( treatment );
        }

        if ( !StringUtils.isBlank( channel.getExtractProtocol() ) ) {
            Treatment treatment = Treatment.Factory.newInstance();
            treatment.setName( sample.getGeoAccession() + " channel " + channel.getChannelNumber() + " extraction" );
            treatment.setDescription( channel.getExtractProtocol() );
            bioMaterial.getTreatments().add( treatment );
        }

        if ( !StringUtils.isBlank( channel.getLabelProtocol() ) ) {
            Treatment treatment = Treatment.Factory.newInstance();
            treatment.setName( sample.getGeoAccession() + " channel " + channel.getChannelNumber() + " label" );
            treatment.setDescription( channel.getLabelProtocol() );
            bioMaterial.getTreatments().add( treatment );
        }

        for ( String characteristic : channel.getCharacteristics() ) {
            Characteristic gemmaChar = Characteristic.Factory.newInstance();

            characteristic = trimString( characteristic );

            gemmaChar.setCategory( "GEO Sample characteristic" );
            gemmaChar.setValue( characteristic );
            bioMaterial.getCharacteristics().add( gemmaChar );
        }

        if ( channel.getSourceName() != null ) {
            Characteristic sourceChar = Characteristic.Factory.newInstance();
            sourceChar.setCategory( "GEO Sample source" );
            String characteristic = trimString( channel.getSourceName() );
            sourceChar.setValue( characteristic );
            bioMaterial.getCharacteristics().add( sourceChar );
        }

        if ( channel.getOrganism() != null ) {
            Taxon taxon = Taxon.Factory.newInstance();
            taxon.setScientificName( channel.getOrganism() );
            bioMaterial.setSourceTaxon( taxon );
        }

        if ( channel.getMolecule() != null ) {
            Characteristic moleculeChar = Characteristic.Factory.newInstance();
            moleculeChar.setCategory( "GEO Sample molecule" );
            String characteristic = trimString( channel.getMolecule().toString() );
            moleculeChar.setValue( characteristic );
            bioMaterial.getCharacteristics().add( moleculeChar );
        }

        if ( channel.getLabel() != null ) {
            Characteristic labelChar = Characteristic.Factory.newInstance();
            labelChar.setCategory( "GEO Sample label" );
            String characteristic = trimString( channel.getLabel() );
            labelChar.setValue( characteristic );
            bioMaterial.getCharacteristics().add( labelChar );
        }

        return bioMaterial;
    }

    private String trimString( String characteristic ) {
        if ( characteristic.length() > 255 ) {
            log.warn( "** Characteristic too long: " + characteristic + " - will truncate - ****" );
            characteristic = characteristic.substring( 0, 199 ) + " (truncated at 200 characters)";
        }
        return characteristic;
    }

    /**
     * @param contact
     * @return
     */
    private Person convertContact( GeoContact contact ) {
        Person result = Person.Factory.newInstance();
        result.setAddress( contact.getCity() + " " + contact.getState() + " " + contact.getCountry() + " "
                + contact.getPostCode() );
        result.setPhone( contact.getPhone() );
        result.setName( contact.getName() );
        result.setEmail( contact.getEmail() );
        result.setFax( contact.getFax() );
        result.setURL( contact.getWebLink() );

        return result;
    }

    /**
     * Take contact and contributer information from a GeoSeries and put it in the ExpressionExperiment.
     * 
     * @param series
     * @param expExp
     */
    @SuppressWarnings("unchecked")
    private void convertContacts( GeoSeries series, ExpressionExperiment expExp ) {
        expExp.getInvestigators().add( convertContact( series.getContact() ) );
        if ( series.getContributers().size() > 0 ) {
            expExp.setDescription( expExp.getDescription() + "\nContributers: " );
            for ( GeoContact contributer : series.getContributers() ) {
                expExp.setDescription( expExp.getDescription() + " " + contributer.getName() );
                expExp.getInvestigators().add( convertContact( contributer ) );
            }
            expExp.setDescription( expExp.getDescription() + "\n" );
        }
    }

    /**
     * Convert a vector of strings into a byte[] for saving in the database. . Blanks(missing values) are treated as NAN
     * (double), 0 (integer), false (booleans) or just empty strings (strings). Other invalid values are treated the
     * same way as missing data (to keep the parser from failing when dealing with strange GEO files that have values
     * like "Error" for an expression value).
     * 
     * @param vector of Strings to be converted to primitive values (double, int etc)
     * @param qt The quantitation type for the values to be converted.
     * @return
     */
    protected byte[] convertData( List<Object> vector, QuantitationType qt ) {

        if ( vector == null || vector.size() == 0 ) return null;

        boolean containsAtLeastOneNonNull = false;
        for ( Object string : vector ) {
            if ( string != null ) {
                containsAtLeastOneNonNull = true;
                break;
            }
        }

        if ( !containsAtLeastOneNonNull ) {
            if ( log.isDebugEnabled() ) {
                log.debug( "No data for " + qt + " in vector of length " + vector.size() );
            }
            return null;
        }

        List<Object> toConvert = new ArrayList<Object>();
        PrimitiveType pt = qt.getRepresentation();
        int numMissing = 0;
        for ( Object rawValue : vector ) {
            if ( rawValue == null ) {
                numMissing++;
                handleMissing( toConvert, pt );
            } else if ( rawValue instanceof String ) { // needs to be coverted.
                try {
                    if ( pt.equals( PrimitiveType.DOUBLE ) ) {
                        toConvert.add( Double.parseDouble( ( String ) rawValue ) );
                    } else if ( pt.equals( PrimitiveType.STRING ) || pt.equals( PrimitiveType.STRINGARRAY ) ) {
                        // FIXME BUG 741 - converter makes it a tabbed string, which is a no-no.
                        toConvert.add( ( String ) rawValue );
                    } else if ( pt.equals( PrimitiveType.CHAR ) ) {
                        if ( ( ( String ) rawValue ).length() != 1 ) {
                            throw new IllegalStateException( "Attempt to cast a string of length "
                                    + ( ( String ) rawValue ).length() + " to a char" );
                        }
                        toConvert.add( ( Character ) ( ( String ) rawValue ).toCharArray()[0] );
                    } else if ( pt.equals( PrimitiveType.INT ) ) {
                        toConvert.add( Integer.parseInt( ( String ) rawValue ) );
                    } else if ( pt.equals( PrimitiveType.BOOLEAN ) ) {
                        toConvert.add( Boolean.parseBoolean( ( String ) rawValue ) );

                    } else {
                        throw new UnsupportedOperationException( "Data vectors of type " + pt + " not supported" );
                    }
                } catch ( NumberFormatException e ) {
                    numMissing++;
                    handleMissing( toConvert, pt );
                } catch ( NullPointerException e ) {
                    numMissing++;
                    handleMissing( toConvert, pt );
                }
            } else { // use as is.
                toConvert.add( rawValue );
            }
        }

        if ( numMissing == vector.size() ) {
            return null;
        }

        return byteArrayConverter.toBytes( toConvert.toArray() );
    }

    /**
     * Often-needed generation of a valid databaseentry object.
     * 
     * @param geoData
     * @return
     */
    private DatabaseEntry convertDatabaseEntry( GeoData geoData ) {
        DatabaseEntry result = DatabaseEntry.Factory.newInstance();

        initGeoExternalDatabase();

        result.setExternalDatabase( this.geoDatabase );
        result.setAccession( geoData.getGeoAccession() );

        return result;
    }

    /**
     * @param geoDataset
     */
    private ExpressionExperiment convertDataset( GeoDataset geoDataset ) {

        if ( geoDataset.getSeries().size() == 0 ) {
            throw new IllegalArgumentException( "GEO Dataset must have associated series" );
        }

        if ( geoDataset.getSeries().size() > 1 ) {
            throw new UnsupportedOperationException( "GEO Dataset can only be associated with one series" );
        }

        Collection<ExpressionExperiment> results = this.convertSeries( geoDataset.getSeries().iterator().next() );
        assert results.size() == 1; // unless we have multiple species, not possible.
        return results.iterator().next();
    }

    /**
     * @param dataset
     * @param expExp
     */
    private ExpressionExperiment convertDataset( GeoDataset geoDataset, ExpressionExperiment expExp ) {

        /*
         * First figure out of there are any samples for this data set. It could be that they were duplicates of ones
         * found in other series, so were skipped. See GeoDatasetService
         */
        if ( this.getDatasetSamples( geoDataset ).size() == 0 ) {
            log.info( "No samples remain for " + geoDataset + ", nothing to do" );
            return expExp;
        }

        log.info( "Converting dataset:" + geoDataset );

        convertDatasetDescriptions( geoDataset, expExp );

        GeoPlatform platform = geoDataset.getPlatform();
        ArrayDesign ad = seenPlatforms.get( platform.getGeoAccession() );
        if ( ad == null ) {
            throw new IllegalStateException( "ArrayDesigns must be converted before datasets - didn't find "
                    + geoDataset.getPlatform() );
        }

        ad.setDescription( ad.getDescription() + "\nFrom " + platform.getGeoAccession() + "\nLast Updated: "
                + platform.getLastUpdateDate() );

        LocalFile arrayDesignRawFile = convertSupplementaryFileToLocalFile( platform );
        if ( arrayDesignRawFile != null ) {
            Collection<LocalFile> arrayDesignLocalFiles = ad.getLocalFiles();
            if ( arrayDesignLocalFiles == null ) {
                arrayDesignLocalFiles = new HashSet<LocalFile>();
            }
            arrayDesignLocalFiles.add( arrayDesignRawFile );
            ad.setLocalFiles( arrayDesignLocalFiles );
        }

        convertDataSetDataVectors( geoDataset.getSeries().iterator().next().getValues(), geoDataset, expExp );

        convertSubsetAssociations( expExp, geoDataset );
        return expExp;

    }

    /**
     * Use this when we don't have a GDS for a GSE.
     * 
     * @param geoSeries
     * @param expExp
     */
    private void convertSeriesDataVectors( GeoSeries geoSeries, ExpressionExperiment expExp ) {
        /*
         * Tricky thing is that series contains data from multiple platforms.
         */
        DatasetCombiner datasetCombiner = new DatasetCombiner();
        Map<GeoPlatform, List<GeoSample>> platformSamples = datasetCombiner.getPlatformSampleMap( geoSeries );

        for ( GeoPlatform platform : platformSamples.keySet() ) {
            List<GeoSample> samples = platformSamples.get( platform );
            log.info( samples.size() + " samples on " + platform );
            convertVectorsForPlatform( geoSeries.getValues(), expExp, samples, platform );
            geoSeries.getValues().clear( platform );
        }

    }

    /**
     * Convert the GEO data into DesignElementDataVectors associated with the ExpressionExperiment
     * 
     * @param geoDataset Source of the data
     * @param expExp ExpressionExperiment to fill in.
     */
    private void convertDataSetDataVectors( GeoValues values, GeoDataset geoDataset, ExpressionExperiment expExp ) {
        List<GeoSample> datasetSamples = new ArrayList<GeoSample>( getDatasetSamples( geoDataset ) );
        log.info( datasetSamples.size() + " samples in " + geoDataset );
        GeoPlatform geoPlatform = geoDataset.getPlatform();

        convertVectorsForPlatform( values, expExp, datasetSamples, geoPlatform );

        values.clear( geoPlatform );
    }

    /**
     * @param expExp
     * @param datasetSamples
     * @param geoPlatform
     */
    private void convertVectorsForPlatform( GeoValues values, ExpressionExperiment expExp,
            List<GeoSample> datasetSamples, GeoPlatform geoPlatform ) {
        assert datasetSamples.size() > 0 : "No samples in dataset";

        log.info( "Converting vectors for " + geoPlatform.getGeoAccession() + ", " + datasetSamples.size()
                + " samples." );

        BioAssayDimension bioAssayDimension = convertGeoSampleList( datasetSamples, expExp );

        if ( bioAssayDimension.getBioAssays().size() == 0 )
            throw new IllegalStateException( "No bioAssays in the BioAssayDimension" );

        sanityCheckQuantitationTypes( datasetSamples );

        List<String> quantitationTypes = datasetSamples.iterator().next().getColumnNames();
        List<String> quantitationTypeDescriptions = datasetSamples.iterator().next().getColumnDescriptions();
        boolean first = true;

        /*
         * For the data that are put in 'datasets' (GDS), we know the type of data, but it can be misleading (e.g., Affy
         * data is 'counts'). For others we just have free text in the column descriptions
         */

        int quantitationTypeIndex = 0;
        for ( String quantitationType : quantitationTypes ) {

            // skip the first quantitationType, it's the ID or ID_REF.
            if ( first ) {
                first = false;
                // quantitationTypeIndex++; // don't count the first column, don't increment.
                continue;
            }

            log.info( "Processing " + quantitationType + " (column=" + quantitationTypeIndex + ")" );

            Map<String, List<Object>> dataVectors = makeDataVectors( values, datasetSamples, quantitationTypeIndex );

            if ( dataVectors == null ) {
                log.info( "No data for " + quantitationType + " (column=" + quantitationTypeIndex + ")" );
                quantitationTypeIndex++;
                continue;
            } else {
                log.info( "Got " + dataVectors.size() + " data vectors for " + quantitationType + " (column="
                        + quantitationTypeIndex + ")" );
            }

            QuantitationType qt = QuantitationType.Factory.newInstance();
            qt.setName( quantitationType );
            String description = quantitationTypeDescriptions.get( quantitationTypeIndex + 1 );
            qt.setDescription( description );
            QuantitationTypeParameterGuesser.guessQuantitationTypeParameters( qt, quantitationType, description );

            expExp.getQuantitationTypes().add( qt );

            int count = 0;
            for ( String designElementName : dataVectors.keySet() ) {
                List<Object> dataVector = dataVectors.get( designElementName );
                if ( dataVector == null || dataVector.size() == 0 ) continue;

                DesignElementDataVector vector = convertDesignElementDataVector( geoPlatform, expExp,
                        bioAssayDimension, designElementName, dataVector, qt );

                if ( vector == null ) {
                    if ( log.isDebugEnabled() )
                        log.debug( "Null vector for DE=" + designElementName + " QT=" + quantitationType );
                    continue;
                }

                if ( log.isTraceEnabled() ) {
                    log.trace( designElementName + " " + qt.getName() + " " + qt.getRepresentation() + " "
                            + dataVector.size() + " elements in vector" );
                }

                expExp.getDesignElementDataVectors().add( vector );

                if ( log.isDebugEnabled() && ++count % LOGGING_VECTOR_COUNT_UPDATE == 0 ) {
                    log.debug( count + " Data vectors added" );
                }
            }
            if ( log.isDebugEnabled() ) {
                log.debug( count + " Data vectors added for '" + quantitationType + "'" );
            }
            quantitationTypeIndex++;
        }
        log.info( "Total of " + expExp.getDesignElementDataVectors().size() + " vectors so far..." );
    }

    /**
     * @param geoDataset
     * @param expExp
     */
    private void convertDatasetDescriptions( GeoDataset geoDataset, ExpressionExperiment expExp ) {
        if ( StringUtils.isEmpty( expExp.getDescription() ) ) {
            expExp.setDescription( geoDataset.getDescription() ); // probably not empty.
        }

        expExp.setDescription( expExp.getDescription() + "\nIncludes " + geoDataset.getGeoAccession() + ".\n" );
        if ( StringUtils.isNotEmpty( geoDataset.getUpdateDate() ) ) {
            expExp.setDescription( expExp.getDescription() + " Update date: " + geoDataset.getUpdateDate() + ".\n" );
        }

        if ( StringUtils.isEmpty( expExp.getName() ) ) {
            expExp.setName( geoDataset.getTitle() );
        } else {
            expExp.setDescription( expExp.getDescription() + " Dataset description " + geoDataset.getGeoAccession()
                    + ": " + geoDataset.getTitle() + ".\n" );
        }
    }

    /**
     * @param geoDataset
     * @param expExp
     * @param bioAssayDimension
     * @param designElementName
     * @param dataVector to convert.
     * @return vector, or null if the dataVector was null or empty.
     */
    private DesignElementDataVector convertDesignElementDataVector( GeoPlatform geoPlatform,
            ExpressionExperiment expExp, BioAssayDimension bioAssayDimension, String designElementName,
            List<Object> dataVector, QuantitationType qt ) {

        if ( dataVector == null || dataVector.size() == 0 ) return null;

        byte[] blob = convertData( dataVector, qt );
        if ( blob == null ) { // all missing etc.
            if ( log.isDebugEnabled() ) log.debug( "All missing values for DE=" + designElementName + " QT=" + qt );
            return null;
        }
        if ( log.isDebugEnabled() ) {
            log.debug( blob.length + " bytes for " + dataVector.size() + " raw elements" );
        }

        CompositeSequence compositeSequence = platformDesignElementMap.get(
                convertPlatform( geoPlatform ).getShortName() ).get( designElementName );

        if ( compositeSequence == null ) {
            assert compositeSequence != null : "No composite sequence " + designElementName;
        }

        if ( compositeSequence.getBiologicalCharacteristic() != null
                && compositeSequence.getBiologicalCharacteristic().getSequenceDatabaseEntry() != null ) {
            assert compositeSequence.getBiologicalCharacteristic().getSequenceDatabaseEntry().getExternalDatabase()
                    .getName() != null;
        }

        DesignElementDataVector vector = DesignElementDataVector.Factory.newInstance();
        vector.setDesignElement( compositeSequence );
        vector.setExpressionExperiment( expExp );

        vector.setBioAssayDimension( bioAssayDimension );
        vector.setQuantitationType( qt );
        vector.setData( blob );
        return vector;
    }

    /**
     * @param datasetSamples List of GeoSamples to be matched up with BioAssays.
     * @param expExp ExpresssionExperiment
     * @return BioAssayDimension representing the samples.
     */
    @SuppressWarnings("unchecked")
    private BioAssayDimension convertGeoSampleList( List<GeoSample> datasetSamples, ExpressionExperiment expExp ) {
        BioAssayDimension resultBioAssayDimension = BioAssayDimension.Factory.newInstance();

        StringBuilder bioAssayDimName = new StringBuilder();
        Collections.sort( datasetSamples );
        for ( GeoSample sample : datasetSamples ) {
            boolean found = false;
            String sampleAcc = sample.getGeoAccession();
            bioAssayDimName.append( sampleAcc + "," ); // this is rather silly!
            found = matchSampleToBioAssay( expExp, resultBioAssayDimension, sampleAcc );
            if ( !found ) {
                // this is normal because not all headings are
                // sample ids.
                log.warn( "No bioassay match for " + sampleAcc );
            }
        }
        log.info( resultBioAssayDimension.getBioAssays().size() + " Bioassays in biodimension" );
        resultBioAssayDimension.setName( formatName( bioAssayDimName ) );
        resultBioAssayDimension.setDescription( bioAssayDimName.toString() );
        return resultBioAssayDimension;
    }

    /**
     * @param platform
     */
    @SuppressWarnings("unchecked")
    private ArrayDesign convertPlatform( GeoPlatform platform ) {

        if ( seenPlatforms.containsKey( platform.getGeoAccession() ) ) {
            return ( seenPlatforms.get( platform.getGeoAccession() ) );
        }

        if ( isSage( platform ) ) {
            throw new UnsupportedOperationException( "This data set uses SAGE, it cannot be handled yet" );
        }

        log.info( "Converting platform: " + platform.getGeoAccession() );
        ArrayDesign arrayDesign = createMinimalArrayDesign( platform );

        platformDesignElementMap.put( arrayDesign.getShortName(), new HashMap<String, CompositeSequence>() );

        Taxon taxon = convertPlatformOrganism( platform );

        // convert the design element information.
        String identifier = determinePlatformIdentifier( platform );
        Collection<String> externalReferences = determinePlatformExternalReferenceIdentifier( platform );
        String descriptionColumn = determinePlatformDescriptionColumn( platform );
        String sequenceColumn = determinePlatformSequenceColumn( platform );
        ExternalDatabase externalDb = determinePlatformExternalDatabase( platform );

        List<String> identifiers = platform.getColumnData( identifier );
        List<String> descriptions = platform.getColumnData( descriptionColumn );
        List<String> sequences = platform.getColumnData( sequenceColumn );
        List<String> cloneIdentifiers = platform.getColumnData( "CLONE_ID" );

        List<List<String>> externalRefs = null;
        if ( externalReferences != null ) {
            externalRefs = platform.getColumnData( externalReferences );
        }

        assert identifier != null;

        if ( externalRefs != null ) {
            assert externalRefs.iterator().next().size() == identifiers.size() : "Unequal numbers of identifiers and external references! "
                    + externalRefs.iterator().next().size() + " != " + identifiers.size();
        }
        assert cloneIdentifiers == null || cloneIdentifiers.size() == identifiers.size();

        if ( log.isDebugEnabled() ) {
            log.debug( "Converting " + identifiers.size() + " probe identifiers on GEO platform "
                    + platform.getGeoAccession() );
        }

        Iterator<String> descIter = null;

        if ( descriptions != null ) {
            descIter = descriptions.iterator();
        }

        // much faster than hashset to add to when it gets large. (because equals isn't implemented right... FIXME)
        Collection compositeSequences = new ArrayList( 5000 );
        int i = 0; // to get sequences, if we have them, and clone identifiers.
        for ( String id : identifiers ) {
            String externalAccession = null;
            if ( externalRefs != null ) {
                externalAccession = getExternalAccession( externalRefs, i );
            }

            String cloneIdentifier = cloneIdentifiers == null ? null : cloneIdentifiers.get( i );

            String description = "";
            if ( externalAccession != null ) {
                String[] refs = externalAccession.split( "," );
                if ( refs.length > 1 ) {
                    description = "Multiple external sequence references: " + externalAccession + "; ";
                    externalAccession = refs[0];
                }
            }

            if ( descIter != null ) description = description + " " + descIter.next();

            CompositeSequence cs = CompositeSequence.Factory.newInstance();
            cs.setName( id );
            cs.setDescription( description );
            cs.setArrayDesign( arrayDesign );

            BioSequence bs = createMinimalBioSequence( taxon );

            boolean isRefseq = false;
            if ( externalDb.getName().equals( "Genbank" ) && StringUtils.isNotBlank( externalAccession ) ) {
                // http://www.ncbi.nlm.nih.gov/RefSeq/key.html#accessions : "RefSeq accession numbers can be
                // distinguished from GenBank accessions by their prefix distinct format of [2 characters|underbar]"
                isRefseq = externalAccession.matches( "^[A-Z]{2}_" );
            }

            boolean isImage = false;
            if ( StringUtils.isNotBlank( cloneIdentifier ) ) {
                bs.setName( cloneIdentifier );
                isImage = cloneIdentifier.startsWith( "IMAGE" );
            }

            /*
             * If we are given a sequence, we don't need the genbank identifier, which is probably not correct anyway.
             */
            if ( sequences != null && StringUtils.isNotBlank( sequences.get( i ) ) ) {
                bs.setSequence( sequences.get( i ) );
                bs.setIsApproximateLength( false );
                bs.setLength( new Long( bs.getSequence().length() ) );
                bs.setType( SequenceType.DNA );
                bs.setName( id + "_sequence" );
                bs.setDescription( "Sequence provided by manufacturer. "
                        + ( externalAccession != null ? "Used in leiu of " + externalAccession
                                : "No external accession provided" ) );
            } else if ( externalAccession != null && !isRefseq && !isImage && externalDb != null ) {
                /*
                 * We don't use this if we have an IMAGE clone because the accession might be wrong (e.g., for a
                 * Refseq). During persisting the IMAGE clone will be replaced with the 'real' thing.
                 */

                /*
                 * We also don't store them if they are refseq ids, because refseq ids are not actually put on arrays.
                 */

                DatabaseEntry dbe = createDatabaseEntry( externalDb, externalAccession, bs );
                bs.setSequenceDatabaseEntry( dbe );
            }

            /*
             * If we have no basis for describing the sequence, we have to skip it.
             */
            if ( StringUtils.isBlank( externalAccession ) && StringUtils.isBlank( cloneIdentifier ) ) {
                if ( log.isDebugEnabled() ) {
                    log.debug( "Blank external reference and clone id for " + cs + " on " + arrayDesign
                            + ", no biological characteristic can be added." );
                }
            } else {
                cs.setBiologicalCharacteristic( bs );
            }

            compositeSequences.add( cs );

            platformDesignElementMap.get( arrayDesign.getShortName() ).put( id, cs );
            i++;
        }
{'fraction_non_alphanumeric': 0.10024221856663601, 'fraction_numerical': 0.001515485149540821, 'mean_word_length': 2.4093177301832633, 'pattern_counts': {'":': 2, '<': 101, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '19167', 'n_tokens_mistral': 20309, 'n_tokens_neox': 19450, 'n_words': 5039}