<commit_before>/*
 * The Gemma project
 * 
 * Copyright (c) 2007 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.core.analysis.service;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import ubic.basecode.util.FileTools;
import ubic.gemma.core.analysis.expression.diff.DifferentialExpressionAnalyzerService;
import ubic.gemma.core.analysis.preprocess.ExpressionDataMatrixBuilder;
import ubic.gemma.core.analysis.preprocess.filter.FilterConfig;
import ubic.gemma.core.datastructure.matrix.ExperimentalDesignWriter;
import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix;
import ubic.gemma.core.datastructure.matrix.ExpressionDataMatrix;
import ubic.gemma.core.datastructure.matrix.MatrixWriter;
import ubic.gemma.core.expression.experiment.service.ExpressionExperimentService;
import ubic.gemma.model.analysis.expression.diff.ContrastResult;
import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysis;
import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysisResult;
import ubic.gemma.persistence.service.analysis.expression.diff.DifferentialExpressionAnalysisService;
import ubic.gemma.model.analysis.expression.diff.ExpressionAnalysisResultSet;
import ubic.gemma.persistence.service.association.coexpression.CoexpressionService;
import ubic.gemma.persistence.service.association.coexpression.CoexpressionValueObject;
import ubic.gemma.model.common.description.Characteristic;
import ubic.gemma.model.common.quantitationtype.PrimitiveType;
import ubic.gemma.model.common.quantitationtype.QuantitationType;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
import ubic.gemma.persistence.service.expression.bioAssayData.DesignElementDataVectorService;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.expression.experiment.BioAssaySet;
import ubic.gemma.model.expression.experiment.ExperimentalFactor;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.model.expression.experiment.ExpressionExperimentSubSet;
import ubic.gemma.model.expression.experiment.FactorType;
import ubic.gemma.model.expression.experiment.FactorValue;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.persistence.util.DifferentialExpressionAnalysisResultComparator;
import ubic.gemma.persistence.util.EntityUtils;

/**
 * Supports the creation and location of 'flat file' versions of data in the system, for download by users. Files are
 * cached on the filesystem and reused if possible, rather than recreating them every time.
 * <p>
 * 
 * @author paul
 */
@Component
public class ExpressionDataFileServiceImpl implements ExpressionDataFileService {

    private static final String DECIMAL_FORMAT = "%.4g";

    private static Log log = LogFactory.getLog( ArrayDesignAnnotationServiceImpl.class.getName() );

    /**
     * FIXME this is a common chunk of code... should refactor.
     * 
     * @param bas
     * @return
     */
    private static ExpressionExperiment experimentForBioAssaySet( BioAssaySet bas ) {
        ExpressionExperiment ee = null;
        if ( bas instanceof ExpressionExperimentSubSet ) {
            ee = ( ( ExpressionExperimentSubSet ) bas ).getSourceExperiment();
        } else {
            ee = ( ExpressionExperiment ) bas;
        }
        return ee;
    }

    @Autowired
    private ArrayDesignService arrayDesignService;

    @Autowired
    private DesignElementDataVectorService designElementDataVectorService;

    @Autowired
    private DifferentialExpressionAnalysisService differentialExpressionAnalysisService = null;

    @Autowired
    private ExpressionDataMatrixService expressionDataMatrixService;

    @Autowired
    private ExpressionExperimentService expressionExperimentService;

    @Autowired
    private CoexpressionService gene2geneCoexpressionService = null;

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#analysisResultSetsToString(java.util.Collection,
     * java.util.Map, java.lang.StringBuilder)
     */
    @Override
    public void analysisResultSetsToString( Collection<ExpressionAnalysisResultSet> results,
            Map<Long, String[]> geneAnnotations, StringBuilder buf ) {
        Map<Long, StringBuilder> probe2String = new HashMap<Long, StringBuilder>();

        List<DifferentialExpressionAnalysisResult> sortedFirstColumnOfResults = null;

        /*
         * See bug 2085
         */

        for ( ExpressionAnalysisResultSet ears : results ) {
            // ears = differentialExpressionResultService.thaw( ears );
            sortedFirstColumnOfResults = analysisResultSetToString( ears, geneAnnotations, buf, probe2String,
                    sortedFirstColumnOfResults );

        } // ears loop

        buf.append( "\n" );

        if ( sortedFirstColumnOfResults == null ) {
            throw new IllegalStateException( "No results for " );
        }

        // Dump the probe data in the sorted order of the 1st column that we orginally sorted
        for ( DifferentialExpressionAnalysisResult sortedResult : sortedFirstColumnOfResults ) {

            CompositeSequence cs = sortedResult.getProbe();
            StringBuilder sb = probe2String.get( cs.getId() );
            if ( sb == null ) {
                log.warn( "Unable to find probe " + cs.getId() + " in map" );
                break;
            }
            buf.append( sb );
            buf.append( "\n" );

        }
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#analysisResultSetToString(ubic.gemma.model.analysis.
     * expression
     * .diff.ExpressionAnalysisResultSet, java.util.Map, java.lang.StringBuilder, java.util.Map, java.util.List)
     */
    @Override
    public List<DifferentialExpressionAnalysisResult> analysisResultSetToString( ExpressionAnalysisResultSet ears,
            Map<Long, String[]> geneAnnotations, StringBuilder buf, Map<Long, StringBuilder> probe2String,
            List<DifferentialExpressionAnalysisResult> sortedFirstColumnOfResults ) {

        if ( sortedFirstColumnOfResults == null ) { // Sort P values in ears (because 1st column)
            sortedFirstColumnOfResults = new ArrayList<DifferentialExpressionAnalysisResult>( ears.getResults() );
            Collections.sort( sortedFirstColumnOfResults,
                    DifferentialExpressionAnalysisResultComparator.Factory.newInstance() );
        }

        // Generate a description of the factors involved "(factor1:factor2: .... :factorN)"
        String factorColumnName = "(";
        for ( ExperimentalFactor ef : ears.getExperimentalFactors() ) {
            factorColumnName += ef.getName() + ":";
        }
        factorColumnName = StringUtils.removeEnd( factorColumnName, ":" ) + ")";

        // Generate headers
        buf.append( "\tQValue" + factorColumnName );
        buf.append( "\tPValue" + factorColumnName );

        // Generate probe details
        for ( DifferentialExpressionAnalysisResult dear : ears.getResults() ) {
            StringBuilder probeBuffer = new StringBuilder();

            CompositeSequence cs = dear.getProbe();

            // Make a hashmap so we can organize the data by probe with factors as colums
            // Need to cache the information untill we have it organized in the correct format to write
            Long csid = cs.getId();
            if ( probe2String.containsKey( csid ) ) {
                probeBuffer = probe2String.get( csid );
            } else {// no entry for probe yet
                probeBuffer.append( cs.getName() );
                if ( geneAnnotations.containsKey( csid ) ) {
                    String[] annotationStrings = geneAnnotations.get( csid );
                    probeBuffer.append( "\t" + annotationStrings[1] + "\t" + annotationStrings[2] );

                    // leaving out Gemma ID, which is annotationStrings[3]
                    if ( annotationStrings.length > 4 ) {
                        // ncbi id.
                        probeBuffer.append( "\t" + annotationStrings[4] );
                    }
                }

                probe2String.put( csid, probeBuffer );
            }

            Double correctedPvalue = dear.getCorrectedPvalue();
            Double pvalue = dear.getPvalue();

            String formattedCP = correctedPvalue == null ? "" : String.format( DECIMAL_FORMAT, correctedPvalue );
            String formattedP = pvalue == null ? "" : String.format( DECIMAL_FORMAT, pvalue );
            probeBuffer.append( "\t" + formattedCP + "\t" + formattedP );

        } // ears.getResults loop
        return sortedFirstColumnOfResults;
    }

    /**
     * @param resultSet
     * @param geneAnnotations
     * @return
     */
    public String analysisResultSetWithContrastsToString( ExpressionAnalysisResultSet resultSet,
            Map<Long, String[]> geneAnnotations ) {
        StringBuilder buf = new StringBuilder();

        ExperimentalFactor ef = resultSet.getExperimentalFactors().iterator().next();

        // This is a bit risky, we're only looking at the first one. But this is how we do it for the header.
        boolean hasNCBIIDs = !geneAnnotations.isEmpty() && geneAnnotations.values().iterator().next().length > 4;

        if ( ef.getType().equals( FactorType.CONTINUOUS ) ) {

            buf.append( "\tCoefficient (" + ef.getName() + ")\tPValue (" + ef.getName() + ")\n" );

            for ( DifferentialExpressionAnalysisResult dear : resultSet.getResults() ) {
                StringBuilder rowBuffer = new StringBuilder();

                if ( geneAnnotations.isEmpty() ) {
                    rowBuffer.append( dear.getProbe().getName() );
                } else {
                    addGeneAnnotationsToLine( rowBuffer, dear, hasNCBIIDs, geneAnnotations );
                }

                assert dear.getContrasts().size() == 1;

                ContrastResult contrast = dear.getContrasts().iterator().next();

                Double coef = contrast.getCoefficient();
                Double pValue = contrast.getPvalue();
                String formattedPvalue = pValue == null ? "" : String.format( DECIMAL_FORMAT, pValue );
                String formattedcoef = coef == null ? "" : String.format( DECIMAL_FORMAT, coef );
                String contrastData = "\t" + formattedcoef + "\t" + formattedPvalue;

                rowBuffer.append( contrastData );

                buf.append( rowBuffer.toString() + '\n' );
            }

        } else {

            Long baselineId = resultSet.getBaselineGroup().getId();
            List<Long> factorValueIdOrder = new ArrayList<Long>();
            for ( FactorValue factorValue : ef.getFactorValues() ) {
                if ( factorValue.getId() == baselineId ) continue;
                factorValueIdOrder.add( factorValue.getId() );
                // Generate column headers.
                buf.append( "\tFoldChage (" + getFactorValueString( factorValue ) + ")" );
                buf.append( "\tPValue (" + getFactorValueString( factorValue ) + ")" );
            }

            buf.append( '\n' );

            // Generate probe details
            for ( DifferentialExpressionAnalysisResult dear : resultSet.getResults() ) {
                StringBuilder rowBuffer = new StringBuilder();

                addGeneAnnotationsToLine( rowBuffer, dear, hasNCBIIDs, geneAnnotations );

                Map<Long, String> factorValueIdToData = new HashMap<Long, String>();
                // I don't think we can expect them in the same order.
                for ( ContrastResult contrast : dear.getContrasts() ) {
                    Double foldChange = contrast.getLogFoldChange();
                    Double pValue = contrast.getPvalue();
                    String formattedPvalue = pValue == null ? "" : String.format( DECIMAL_FORMAT, pValue );
                    String formattedFoldChange = foldChange == null ? "" : String.format( DECIMAL_FORMAT, foldChange );
                    String contrastData = "\t" + formattedFoldChange + "\t" + formattedPvalue;
                    assert contrast.getFactorValue() != null;
                    factorValueIdToData.put( contrast.getFactorValue().getId(), contrastData );
                }

                // Get them in the right order.
                for ( Long factorValueId : factorValueIdOrder ) {
                    String s = factorValueIdToData.get( factorValueId );
                    if ( s == null ) s = "";
                    rowBuffer.append( s );
                }

                buf.append( rowBuffer.toString() + '\n' );

            } // resultSet.getResults() loop
        }
        return buf.toString();
    }

    /**
     * @param rowBuffer
     * @param dear
     * @param hasNCBIIDs Whether the annotations have the NCBI gene ids. This is based on just peeking at one, so it
     *        might be wrong! But the format will be okay.
     * @param geneAnnotations
     */
    private void addGeneAnnotationsToLine( StringBuilder rowBuffer, DifferentialExpressionAnalysisResult dear,
            boolean hasNCBIIDs, Map<Long, String[]> geneAnnotations ) {
        CompositeSequence cs = dear.getProbe();
        Long csid = cs.getId();
        rowBuffer.append( cs.getName() );
        if ( geneAnnotations.containsKey( csid ) ) {
            String[] annotationStrings = geneAnnotations.get( csid );
            rowBuffer.append( "\t" + annotationStrings[1] + "\t" + annotationStrings[2] );

            // leaving out Gemma ID, which is annotationStrings[3]
            if ( hasNCBIIDs ) {
                // ncbi id.
                rowBuffer.append( "\t" + annotationStrings[4] );
            }
        } else {
            rowBuffer.append( "\t\t" );
            if ( hasNCBIIDs ) {
                rowBuffer.append( "\t" );
            }
        }
    }

    @Override
    public void deleteAllFiles( ExpressionExperiment ee ) throws IOException {
        ee = this.expressionExperimentService.thawLite( ee );

        // data files.
        deleteAndLog( getOutputFile( ee, true ) );
        deleteAndLog( getOutputFile( ee, false ) );

        // diff ex files
        Collection<DifferentialExpressionAnalysis> analyses = this.differentialExpressionAnalysisService
                .getAnalyses( ee );
        for ( DifferentialExpressionAnalysis analysis : analyses ) {
            deleteDiffExArchiveFile( analysis );
        }

        // coexpression file
        deleteAndLog( getOutputFile( getCoexpressionDataFilename( ee ) ) );

        // design file
        deleteAndLog( getOutputFile( getDesignFileName( ee, DATA_FILE_SUFFIX_COMPRESSED ) ) );
    }

    /**
     * @param analysis
     */
    @Override
    public void deleteDiffExArchiveFile( DifferentialExpressionAnalysis analysis ) throws IOException {
        String filename = getDiffExArchiveFileName( analysis );
        deleteAndLog( getOutputFile( filename ) );
    }

    @Autowired
    private DifferentialExpressionAnalyzerService analyzerService;

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDiffExpressionDataFile(ubic.gemma.model.
     * analysis
     * .expression.diff.DifferentialExpressionAnalysis, boolean)
     */
    @Override
    public File getDiffExpressionAnalysisArchiveFile( Long analysisId, boolean forceCreate ) {
        DifferentialExpressionAnalysis analysis = this.differentialExpressionAnalysisService.load( analysisId );

        String filename = getDiffExArchiveFileName( analysis );
        File f = getOutputFile( filename );

        if ( !forceCreate && f.canRead() ) {
            log.info( f + " exists, not regenerating" );
            return f;
        }

        analysis = this.differentialExpressionAnalysisService.thawFully( analysis );
        BioAssaySet experimentAnalyzed = analysis.getExperimentAnalyzed();

        /*
         * Decide if we need to extend the analysis first.
         */
        boolean extend = false;
        for ( ExpressionAnalysisResultSet rs : analysis.getResultSets() ) {
            if ( rs.getQvalueThresholdForStorage() != null && rs.getQvalueThresholdForStorage() < 1.0 ) {
                extend = true;
                break;
            }
        }

        if ( extend ) {
            log.info( "Extending an existing analysis to incldue all results" );

            ExpressionExperiment ee = experimentForBioAssaySet( experimentAnalyzed );

            Collection<ExpressionAnalysisResultSet> updatedResultSets = analyzerService.extendAnalysis( ee, analysis );

            boolean added = analysis.getResultSets().addAll( updatedResultSets );
            assert !added : "Should have simply replaced";
        }

        try {
            analysis = this.differentialExpressionAnalysisService.thawFully( analysis );
            writeDiffExArchiveFile( experimentAnalyzed, analysis );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }

        return f;
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileService#getDiffExpressionAnalysisArchiveFile(ubic.gemma.model.
     * expression.experiment.BioAssaySet, ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysis,
     * java.util.Collection)
     */
    @Override
    public File getDiffExpressionAnalysisArchiveFile( BioAssaySet experimentAnalyzed,
            DifferentialExpressionAnalysis analysis, Collection<ExpressionAnalysisResultSet> resultSets ) {

        assert analysis.getId() != null;

        Collection<ArrayDesign> arrayDesigns = this.expressionExperimentService
                .getArrayDesignsUsed( experimentAnalyzed );
        Map<Long, String[]> geneAnnotations = this.getGeneAnnotationsAsStrings( arrayDesigns );
        String filename = getDiffExArchiveFileName( analysis );
        File f = getOutputFile( filename );

        // We create .zip file with analysis results.
        log.info( "Creating new Differential Expression data archive file: " + f.getName() );
        try (ZipOutputStream zipOut = new ZipOutputStream( new FileOutputStream( f ) );) {

            // Add gene x factor analysis results file.
            zipOut.putNextEntry( new ZipEntry( "analysis.data.txt" ) );

            StringBuilder buf = new StringBuilder();
            buf.append( makeDiffExpressionFileHeader( analysis, resultSets, geneAnnotations ) );
            analysisResultSetsToString( resultSets, geneAnnotations, buf );
            String analysisData = buf.toString();

            zipOut.write( analysisData.getBytes() );
            zipOut.closeEntry();

            // Add a file for each result set with contrasts information.
            for ( ExpressionAnalysisResultSet resultSet : resultSets ) {
                if ( resultSet.getExperimentalFactors().size() > 1 ) {
                    continue; // Skip interactions.
                }

                String resultSetData = convertDiffExpressionResultSetData( resultSet, geneAnnotations );
                zipOut.putNextEntry( new ZipEntry( "resultset_" + resultSet.getId() + ".data.txt" ) );
                zipOut.write( resultSetData.getBytes() );
                zipOut.closeEntry();
            }

            zipOut.close();
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
        log.info( "Full results are stored in " + f.getAbsolutePath() );
        return f;
    }

    /**
     * @param experimentAnalyzed
     * @param analysis
     */
    @Override
    public void writeDiffExArchiveFile( BioAssaySet experimentAnalyzed, DifferentialExpressionAnalysis analysis )
            throws IOException {
        Collection<ArrayDesign> arrayDesigns = this.expressionExperimentService
                .getArrayDesignsUsed( experimentAnalyzed );
        Map<Long, String[]> geneAnnotations = this.getGeneAnnotationsAsStrings( arrayDesigns );
        String filename = getDiffExArchiveFileName( analysis );
        File f = getOutputFile( filename );

        // We create .zip file with analysis results.
        log.info( "Creating new Differential Expression data archive file: " + f.getName() );
        try (ZipOutputStream zipOut = new ZipOutputStream( new FileOutputStream( f ) );) {

            // Add gene x factor analysis results file.
            zipOut.putNextEntry( new ZipEntry( "analysis.data.txt" ) );
            String analysisData = convertDiffExpressionAnalysisData( analysis, geneAnnotations );
            zipOut.write( analysisData.getBytes() );
            zipOut.closeEntry();

            // Add a file for each result set with contrasts information.
            for ( ExpressionAnalysisResultSet resultSet : analysis.getResultSets() ) {
                if ( resultSet.getExperimentalFactors().size() > 1 ) {
                    continue; // Skip interactions.
                }

                assert resultSet.getQvalueThresholdForStorage() == null
                        || resultSet.getQvalueThresholdForStorage() == 1.0;

                String resultSetData = convertDiffExpressionResultSetData( resultSet, geneAnnotations );
                zipOut.putNextEntry( new ZipEntry( "resultset_" + resultSet.getId() + ".data.txt" ) );
                zipOut.write( resultSetData.getBytes() );
                zipOut.closeEntry();
            }
        }
    }

    @Override
    public File getOutputFile( ExpressionExperiment ee, boolean filtered ) {
        return getOutputFile( ee, filtered, true, false );
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#getOutputFile(ubic.gemma.model.expression.experiment.
     * ExpressionExperiment, boolean)
     */
    @Override
    public File getOutputFile( ExpressionExperiment ee, boolean filtered, boolean compressed, boolean temporary ) {
        String filteredAdd = "";
        if ( !filtered ) {
            filteredAdd = ".unfilt";
        }
        String suffix;

        if ( compressed ) {
            suffix = DATA_FILE_SUFFIX_COMPRESSED;
        } else {
            suffix = DATA_FILE_SUFFIX;
        }

        String filename = getDataFileName( ee, filteredAdd, suffix );

        // randomize file name if temporary in case of access by more than one user at once
        if ( temporary ) {

            filename = RandomStringUtils.randomAlphabetic( 6 ) + filename;

        }

        return getOutputFile( filename, temporary );
    }

    @Override
    public File getOutputFile( String filename ) {
        return getOutputFile( filename, false );

    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#getOutputFile(java.lang.String)
     */
    @Override
    public File getOutputFile( String filename, boolean temporary ) {
        String fullFilePath;
        if ( temporary ) {
            fullFilePath = TMP_DATA_DIR + filename;
        } else {
            fullFilePath = DATA_DIR + filename;
        }
        File f = new File( fullFilePath );

        if ( f.exists() ) {
            return f;
        }

        File parentDir = f.getParentFile();
        if ( !parentDir.exists() ) parentDir.mkdirs();
        return f;
    }

    @Override
    public File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName, boolean compress )
            throws IOException {
        File f = new File( fileName );
        return writeDataFile( ee, filtered, f, compress );
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateCoexpressionDataFile(ubic.gemma.model.
     * expression
     * .experiment.ExpressionExperiment, boolean)
     */
    @Override
    public File writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean forceWrite ) {

        ee = expressionExperimentService.thawLite( ee );

        String getCoexpressionDataFilename = getCoexpressionDataFilename( ee );
        String filename = getCoexpressionDataFilename;
        try {
            File f = getOutputFile( filename );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new Co-Expression data file: " + f.getName() );
            writeCoexpressionData( f, ee );
            return f;
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDataFile(ubic.gemma.model.expression.
     * experiment
     * .ExpressionExperiment, boolean, boolean)
     */
    @Override
    public File writeOrLocateDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) {

        try {
            File f = getOutputFile( ee, filtered );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }
            return writeDataFile( ee, filtered, f, true );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    @Override
    public File writeTemporaryDataFile( ExpressionExperiment ee, boolean filtered ) {

        try {

            File f = getOutputFile( ee, filtered, false, true );

            return writeDataFile( ee, filtered, f, false );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDataFile(ubic.gemma.model.common.
     * quantitationtype
     * .QuantitationType, boolean)
     */
    @Override
    public File writeOrLocateDataFile( QuantitationType type, boolean forceWrite ) {

        try {
            File f = getOutputFile( type );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new quantitation type expression data file: " + f.getName() );

            Collection<? extends DesignElementDataVector> vectors = designElementDataVectorService.find( type );
            Collection<ArrayDesign> arrayDesigns = getArrayDesigns( vectors );
            Map<CompositeSequence, String[]> geneAnnotations = this.getGeneAnnotationsAsStringsByProbe( arrayDesigns );

            if ( vectors.size() == 0 ) {
                log.warn( "No vectors for " + type );
                return null;
            }

            writeVectors( f, type.getRepresentation(), vectors, geneAnnotations );
            return f;
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDesignFile(ubic.gemma.model.expression.
     * experiment
     * .ExpressionExperiment, boolean)
     */
    @Override
    public File writeOrLocateDesignFile( ExpressionExperiment ee, boolean forceWrite ) {

        ee = expressionExperimentService.thawLite( ee );

        String filename = getDesignFileName( ee, DATA_FILE_SUFFIX_COMPRESSED );
        try {
            File f = getOutputFile( filename );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new experimental design file: " + f.getName() );
            return writeDesignMatrix( f, ee, true );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileService#writeTemporaryDesignFile(ubic.gemma.model.expression.
     * experiment
     * .ExpressionExperiment)
     */
    @Override
    public File writeTemporaryDesignFile( ExpressionExperiment ee ) {

        ee = expressionExperimentService.thawLite( ee );

        // not compressed
        String filename = getDesignFileName( ee, DATA_FILE_SUFFIX );

        filename = RandomStringUtils.randomAlphabetic( 6 ) + filename;
        try {
            File f = getOutputFile( filename, true );

            log.info( "Creating new experimental design file: " + f.getName() );
            return writeDesignMatrix( f, ee, true, false );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDiffExpressionDataFiles(ubic.gemma.model.
     * expression.experiment.ExpressionExperiment, boolean)
     */
    @Override
    public Collection<File> writeOrLocateDiffExpressionDataFiles( ExpressionExperiment ee, boolean forceWrite ) {

        ee = this.expressionExperimentService.thawLite( ee );

        Collection<DifferentialExpressionAnalysis> analyses = this.differentialExpressionAnalysisService
                .getAnalyses( ee );

        Collection<File> result = new HashSet<>();
        for ( DifferentialExpressionAnalysis analysis : analyses ) {
            assert analysis.getId() != null;
            result.add( this.getDiffExpressionAnalysisArchiveFile( analysis.getId(), forceWrite ) );
        }

        return result;

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateJSONDataFile(ubic.gemma.model.expression.
     * experiment.ExpressionExperiment, boolean, boolean)
     */
    @Override
    public File writeOrLocateJSONDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) {

        try {
            File f = getOutputFile( ee, filtered );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new JSON expression data file: " + f.getName() );
            ExpressionDataDoubleMatrix matrix = getDataMatrix( ee, filtered, f );

            Collection<ArrayDesign> arrayDesigns = expressionExperimentService.getArrayDesignsUsed( ee );
            Map<Long, Collection<Gene>> geneAnnotations = this.getGeneAnnotations( arrayDesigns );

            writeJson( f, geneAnnotations, matrix );
            return f;
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateJSONDataFile(ubic.gemma.model.common.
     * quantitationtype.QuantitationType, boolean)
     */
    @Override
    public File writeOrLocateJSONDataFile( QuantitationType type, boolean forceWrite ) {

        try {
            File f = getJSONOutputFile( type );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new quantitation type  JSON data file: " + f.getName() );

            Collection<? extends DesignElementDataVector> vectors = designElementDataVectorService.find( type );

            if ( vectors.size() == 0 ) {
                log.warn( "No vectors for " + type );
                return null;
            }

            writeJson( f, type.getRepresentation(), vectors );
            return f;
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    /**
     * Given diff exp analysis and gene annotation generate header and tab delimited data. The output is qValue....
     * 
     * @param analysis
     * @param geneAnnotations
     * @return
     */
    private String convertDiffExpressionAnalysisData( DifferentialExpressionAnalysis analysis,
            Map<Long, String[]> geneAnnotations ) {
        Collection<ExpressionAnalysisResultSet> results = analysis.getResultSets();
        if ( results == null || results.isEmpty() ) {
            log.warn( "No differential expression results found for " + analysis );
            return "";
        }

        StringBuilder buf = new StringBuilder();

        buf.append( makeDiffExpressionFileHeader( analysis, analysis.getResultSets(), geneAnnotations ) );
        analysisResultSetsToString( results, geneAnnotations, buf );

        return buf.toString();
    }

    /**
     * Given result set and gene annotation generate header and tab delimited data. The output is foldChange and pValue
     * associated with each contrast.
     * 
     * @param resultSet
     * @param geneAnnotations
     * @return
     */
    private String convertDiffExpressionResultSetData( ExpressionAnalysisResultSet resultSet,
            Map<Long, String[]> geneAnnotations ) {
        StringBuilder buf = new StringBuilder();
        // Write header.
        buf.append( makeDiffExpressionResultSetFileHeader( resultSet, geneAnnotations ) );
        // Write contrasts data.
        buf.append( analysisResultSetWithContrastsToString( resultSet, geneAnnotations ) );

        return buf.toString();
    }

    /**
     * @param f1
     */
    private void deleteAndLog( File f1 ) {
        if ( f1.canWrite() && f1.delete() ) {
            log.info( "Deleted: " + f1 );
        }
    }

    /**
     * @param vectors
     * @return
     */
    private Collection<ArrayDesign> getArrayDesigns( Collection<? extends DesignElementDataVector> vectors ) {
        Collection<ArrayDesign> ads = new HashSet<ArrayDesign>();
        for ( DesignElementDataVector v : vectors ) {
            ads.add( v.getDesignElement().getArrayDesign() );
        }
        return ads;
    }

    /**
     * @param ee
     * @return
     */
    private String getCoexpressionDataFilename( ExpressionExperiment ee ) {
        return ee.getId() + "_" + FileTools.cleanForFileName( ee.getShortName() ) + "_coExp"
                + DATA_FILE_SUFFIX_COMPRESSED;
    }

    /**
     * @param ee
     * @param filteredAdd
     * @return Name, without full path.
     */
    private String getDataFileName( ExpressionExperiment ee, String filteredAdd, String suffix ) {
        return ee.getId() + "_" + FileTools.cleanForFileName( ee.getShortName() ) + "_expmat" + filteredAdd + suffix;
    }

    /**
     * @param ee
     * @param filtered
     * @param f
     * @return
     */
    private ExpressionDataDoubleMatrix getDataMatrix( ExpressionExperiment ee, boolean filtered, File f ) {

        FilterConfig filterConfig = new FilterConfig();
        filterConfig.setIgnoreMinimumSampleThreshold( true );
        filterConfig.setIgnoreMinimumRowsThreshold( true );
        ee = expressionExperimentService.thawLite( ee );
        ExpressionDataDoubleMatrix matrix;
        if ( filtered ) {
            matrix = expressionDataMatrixService.getFilteredMatrix( ee, filterConfig );
        } else {
            matrix = expressionDataMatrixService.getProcessedExpressionDataMatrix( ee );
        }
        return matrix;
    }

    /**
     * @param ee
     * @return
     */
    private String getDesignFileName( ExpressionExperiment ee, String suffix ) {
        return ee.getId() + "_" + FileTools.cleanForFileName( ee.getShortName() ) + "_expdesign" + suffix;
    }

    /**
     * @param diff
     * @return
     */
    private String getDiffExArchiveFileName( DifferentialExpressionAnalysis diff ) {
        BioAssaySet experimentAnalyzed = diff.getExperimentAnalyzed();

        ExpressionExperiment ee;
        if ( experimentAnalyzed instanceof ExpressionExperiment ) {
            ee = ( ExpressionExperiment ) experimentAnalyzed;
        } else if ( experimentAnalyzed instanceof ExpressionExperimentSubSet ) {
            ExpressionExperimentSubSet subset = ( ExpressionExperimentSubSet ) experimentAnalyzed;
            ee = subset.getSourceExperiment();
        } else {
            throw new UnsupportedOperationException( "Don't know about " + experimentAnalyzed.getClass().getName() );
        }

        return experimentAnalyzed.getId() + "_" + FileTools.cleanForFileName( ee.getShortName() ) + "_diffExpAnalysis_"
                + diff.getId() + DATA_ARCHIVE_FILE_SUFFIX;
    }

    /**
     * @param fv
     * @return
     */
    private String getFactorValueString( FactorValue fv ) {
        if ( fv == null ) return "null";

        if ( fv.getCharacteristics() != null && fv.getCharacteristics().size() > 0 ) {
            String fvString = "";
            for ( Characteristic c : fv.getCharacteristics() ) {
                fvString += c.getValue() + " ";
            }
            return fvString;
        } else if ( fv.getMeasurement() != null ) {
            return fv.getMeasurement().getValue();
        } else if ( fv.getValue() != null && !fv.getValue().isEmpty() ) {
            return fv.getValue();
        } else
            return "absent ";
    }

    /**
     * @param ads
     * @return
     */
    private Map<Long, Collection<Gene>> getGeneAnnotations( Collection<ArrayDesign> ads ) {
        Map<Long, Collection<Gene>> annots = new HashMap<Long, Collection<Gene>>();
        for ( ArrayDesign arrayDesign : ads ) {
            arrayDesign = arrayDesignService.thaw( arrayDesign );
            annots.putAll( ArrayDesignAnnotationServiceImpl.readAnnotationFile( arrayDesign ) );
        }
        return annots;
    }

    /**
     * @param ads
     * @return Map of composite sequence ids to an array of strings: [probe name, genes symbol(s), gene Name(s), gemma
     *         id(s), ncbi id(s)].
     */
    private Map<Long, String[]> getGeneAnnotationsAsStrings( Collection<ArrayDesign> ads ) {
        Map<Long, String[]> annots = new HashMap<Long, String[]>();
        for ( ArrayDesign arrayDesign : ads ) {
            arrayDesign = arrayDesignService.thaw( arrayDesign );
            annots.putAll( ArrayDesignAnnotationServiceImpl.readAnnotationFileAsString( arrayDesign ) );
        }
        return annots;
    }

    /**
     * @param ads
     * @return
     */
    private Map<CompositeSequence, String[]> getGeneAnnotationsAsStringsByProbe( Collection<ArrayDesign> ads ) {
        Map<CompositeSequence, String[]> annots = new HashMap<CompositeSequence, String[]>();
        for ( ArrayDesign arrayDesign : ads ) {
            arrayDesign = arrayDesignService.thaw( arrayDesign );

            Map<Long, CompositeSequence> csidmap = EntityUtils.getIdMap( arrayDesign.getCompositeSequences() );

            Map<Long, String[]> geneAnnots = ArrayDesignAnnotationServiceImpl.readAnnotationFileAsString( arrayDesign );

            for ( Entry<Long, String[]> e : geneAnnots.entrySet() ) {

                if ( !csidmap.containsKey( e.getKey() ) ) {
                    continue;
                }

                annots.put( csidmap.get( e.getKey() ), e.getValue() );

            }

        }
        return annots;
    }

    /**
     * @param type
     * @return
     * @throws IOException
     */
    private File getJSONOutputFile( QuantitationType type ) throws IOException {
        String filename = getJSONOutputFilename( type );
        String fullFilePath = DATA_DIR + filename;

        File f = new File( fullFilePath );

        if ( f.exists() ) {
            log.warn( "Will overwrite existing file " + f );
            f.delete();
        }

        File parentDir = f.getParentFile();
        if ( !parentDir.exists() ) parentDir.mkdirs();
        f.createNewFile();
        return f;
    }

    /**
     * @param type
     * @return Name, without full path.
     */
    private String getJSONOutputFilename( QuantitationType type ) {
        return FileTools.cleanForFileName( type.getName() ) + JSON_FILE_SUFFIX;
    }

    private File getOutputFile( QuantitationType type ) {
        String filename = getOutputFilename( type );
        return getOutputFile( filename );
    }

    /**
     * @param type
     * @return Name, without full path.
     */
    private String getOutputFilename( QuantitationType type ) {
        return type.getId() + "_" + FileTools.cleanForFileName( type.getName() ) + DATA_FILE_SUFFIX_COMPRESSED;
    }

    /**
     * @param analysis
     * @param geneAnnotations
     * @param buf
     * @return header string
     */
    private String makeDiffExpressionFileHeader( DifferentialExpressionAnalysis analysis,
            Collection<ExpressionAnalysisResultSet> resultSets, Map<Long, String[]> geneAnnotations ) {

        differentialExpressionAnalysisService.thaw( analysis ); // bug 4023

        StringBuilder buf = new StringBuilder();

        BioAssaySet bas = analysis.getExperimentAnalyzed();
<commit_msg>Allow diff ex CLI to write files rather than database.(fixes)<commit_after>/*
 * The Gemma project
 * 
 * Copyright (c) 2007 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.core.analysis.service;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import ubic.basecode.util.FileTools;
import ubic.gemma.core.analysis.expression.diff.DifferentialExpressionAnalyzerService;
import ubic.gemma.core.analysis.preprocess.ExpressionDataMatrixBuilder;
import ubic.gemma.core.analysis.preprocess.filter.FilterConfig;
import ubic.gemma.core.datastructure.matrix.ExperimentalDesignWriter;
import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix;
import ubic.gemma.core.datastructure.matrix.ExpressionDataMatrix;
import ubic.gemma.core.datastructure.matrix.MatrixWriter;
import ubic.gemma.core.expression.experiment.service.ExpressionExperimentService;
import ubic.gemma.model.analysis.expression.diff.ContrastResult;
import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysis;
import ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysisResult;
import ubic.gemma.persistence.service.analysis.expression.diff.DifferentialExpressionAnalysisService;
import ubic.gemma.model.analysis.expression.diff.ExpressionAnalysisResultSet;
import ubic.gemma.persistence.service.association.coexpression.CoexpressionService;
import ubic.gemma.persistence.service.association.coexpression.CoexpressionValueObject;
import ubic.gemma.model.common.description.Characteristic;
import ubic.gemma.model.common.quantitationtype.PrimitiveType;
import ubic.gemma.model.common.quantitationtype.QuantitationType;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
import ubic.gemma.persistence.service.expression.bioAssayData.DesignElementDataVectorService;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.expression.experiment.BioAssaySet;
import ubic.gemma.model.expression.experiment.ExperimentalFactor;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.model.expression.experiment.ExpressionExperimentSubSet;
import ubic.gemma.model.expression.experiment.FactorType;
import ubic.gemma.model.expression.experiment.FactorValue;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.persistence.util.DifferentialExpressionAnalysisResultComparator;
import ubic.gemma.persistence.util.EntityUtils;

/**
 * Supports the creation and location of 'flat file' versions of data in the system, for download by users. Files are
 * cached on the filesystem and reused if possible, rather than recreating them every time.
 * <p>
 * 
 * @author paul
 */
@Component
public class ExpressionDataFileServiceImpl implements ExpressionDataFileService {

    private static final String DECIMAL_FORMAT = "%.4g";

    private static Log log = LogFactory.getLog( ArrayDesignAnnotationServiceImpl.class.getName() );

    /**
     * FIXME this is a common chunk of code... should refactor.
     * 
     * @param bas
     * @return
     */
    private static ExpressionExperiment experimentForBioAssaySet( BioAssaySet bas ) {
        ExpressionExperiment ee = null;
        if ( bas instanceof ExpressionExperimentSubSet ) {
            ee = ( ( ExpressionExperimentSubSet ) bas ).getSourceExperiment();
        } else {
            ee = ( ExpressionExperiment ) bas;
        }
        return ee;
    }

    @Autowired
    private ArrayDesignService arrayDesignService;

    @Autowired
    private DesignElementDataVectorService designElementDataVectorService;

    @Autowired
    private DifferentialExpressionAnalysisService differentialExpressionAnalysisService = null;

    @Autowired
    private ExpressionDataMatrixService expressionDataMatrixService;

    @Autowired
    private ExpressionExperimentService expressionExperimentService;

    @Autowired
    private CoexpressionService gene2geneCoexpressionService = null;

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#analysisResultSetsToString(java.util.Collection,
     * java.util.Map, java.lang.StringBuilder)
     */
    @Override
    public void analysisResultSetsToString( Collection<ExpressionAnalysisResultSet> results,
            Map<Long, String[]> geneAnnotations, StringBuilder buf ) {
        Map<Long, StringBuilder> probe2String = new HashMap<Long, StringBuilder>();

        List<DifferentialExpressionAnalysisResult> sortedFirstColumnOfResults = null;

        /*
         * See bug 2085
         */

        for ( ExpressionAnalysisResultSet ears : results ) {
            // ears = differentialExpressionResultService.thaw( ears );
            sortedFirstColumnOfResults = analysisResultSetToString( ears, geneAnnotations, buf, probe2String,
                    sortedFirstColumnOfResults );

        } // ears loop

        buf.append( "\n" );

        if ( sortedFirstColumnOfResults == null ) {
            throw new IllegalStateException( "No results for " );
        }

        // Dump the probe data in the sorted order of the 1st column that we orginally sorted
        for ( DifferentialExpressionAnalysisResult sortedResult : sortedFirstColumnOfResults ) {

            CompositeSequence cs = sortedResult.getProbe();
            StringBuilder sb = probe2String.get( cs.getId() );
            if ( sb == null ) {
                log.warn( "Unable to find probe " + cs.getId() + " in map" );
                break;
            }
            buf.append( sb );
            buf.append( "\n" );

        }
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#analysisResultSetToString(ubic.gemma.model.analysis.
     * expression
     * .diff.ExpressionAnalysisResultSet, java.util.Map, java.lang.StringBuilder, java.util.Map, java.util.List)
     */
    @Override
    public List<DifferentialExpressionAnalysisResult> analysisResultSetToString( ExpressionAnalysisResultSet ears,
            Map<Long, String[]> geneAnnotations, StringBuilder buf, Map<Long, StringBuilder> probe2String,
            List<DifferentialExpressionAnalysisResult> sortedFirstColumnOfResults ) {

        if ( sortedFirstColumnOfResults == null ) { // Sort P values in ears (because 1st column)
            sortedFirstColumnOfResults = new ArrayList<DifferentialExpressionAnalysisResult>( ears.getResults() );
            Collections.sort( sortedFirstColumnOfResults,
                    DifferentialExpressionAnalysisResultComparator.Factory.newInstance() );
        }

        // Generate a description of the factors involved "(factor1:factor2: .... :factorN)"
        String factorColumnName = "(";
        for ( ExperimentalFactor ef : ears.getExperimentalFactors() ) {
            factorColumnName += ef.getName() + ":";
        }
        factorColumnName = StringUtils.removeEnd( factorColumnName, ":" ) + ")";

        // Generate headers
        buf.append( "\tQValue" + factorColumnName );
        buf.append( "\tPValue" + factorColumnName );

        // Generate probe details
        for ( DifferentialExpressionAnalysisResult dear : ears.getResults() ) {
            StringBuilder probeBuffer = new StringBuilder();

            CompositeSequence cs = dear.getProbe();

            // Make a hashmap so we can organize the data by probe with factors as colums
            // Need to cache the information untill we have it organized in the correct format to write
            Long csid = cs.getId();
            if ( probe2String.containsKey( csid ) ) {
                probeBuffer = probe2String.get( csid );
            } else {// no entry for probe yet
                probeBuffer.append( cs.getName() );
                if ( geneAnnotations.containsKey( csid ) ) {
                    String[] annotationStrings = geneAnnotations.get( csid );
                    probeBuffer.append( "\t" + annotationStrings[1] + "\t" + annotationStrings[2] );

                    // leaving out Gemma ID, which is annotationStrings[3]
                    if ( annotationStrings.length > 4 ) {
                        // ncbi id.
                        probeBuffer.append( "\t" + annotationStrings[4] );
                    }
                }

                probe2String.put( csid, probeBuffer );
            }

            Double correctedPvalue = dear.getCorrectedPvalue();
            Double pvalue = dear.getPvalue();

            String formattedCP = correctedPvalue == null ? "" : String.format( DECIMAL_FORMAT, correctedPvalue );
            String formattedP = pvalue == null ? "" : String.format( DECIMAL_FORMAT, pvalue );
            probeBuffer.append( "\t" + formattedCP + "\t" + formattedP );

        } // ears.getResults loop
        return sortedFirstColumnOfResults;
    }

    /**
     * @param resultSet
     * @param geneAnnotations
     * @return
     */
    public String analysisResultSetWithContrastsToString( ExpressionAnalysisResultSet resultSet,
            Map<Long, String[]> geneAnnotations ) {
        StringBuilder buf = new StringBuilder();

        ExperimentalFactor ef = resultSet.getExperimentalFactors().iterator().next();

        // This is a bit risky, we're only looking at the first one. But this is how we do it for the header.
        boolean hasNCBIIDs = !geneAnnotations.isEmpty() && geneAnnotations.values().iterator().next().length > 4;

        if ( ef.getType().equals( FactorType.CONTINUOUS ) ) {

            buf.append( "\tCoefficient (" + ef.getName() + ")\tPValue (" + ef.getName() + ")\n" );

            for ( DifferentialExpressionAnalysisResult dear : resultSet.getResults() ) {
                StringBuilder rowBuffer = new StringBuilder();

                if ( geneAnnotations.isEmpty() ) {
                    rowBuffer.append( dear.getProbe().getName() );
                } else {
                    addGeneAnnotationsToLine( rowBuffer, dear, hasNCBIIDs, geneAnnotations );
                }

                assert dear.getContrasts().size() == 1;

                ContrastResult contrast = dear.getContrasts().iterator().next();

                Double coef = contrast.getCoefficient();
                Double pValue = contrast.getPvalue();
                String formattedPvalue = pValue == null ? "" : String.format( DECIMAL_FORMAT, pValue );
                String formattedcoef = coef == null ? "" : String.format( DECIMAL_FORMAT, coef );
                String contrastData = "\t" + formattedcoef + "\t" + formattedPvalue;

                rowBuffer.append( contrastData );

                buf.append( rowBuffer.toString() + '\n' );
            }

        } else {

            Long baselineId = resultSet.getBaselineGroup().getId();
            List<Long> factorValueIdOrder = new ArrayList<Long>();
            for ( FactorValue factorValue : ef.getFactorValues() ) {
                if ( factorValue.getId() == baselineId ) continue;
                factorValueIdOrder.add( factorValue.getId() );
                // Generate column headers.
                buf.append( "\tFoldChage (" + getFactorValueString( factorValue ) + ")" );
                buf.append( "\tPValue (" + getFactorValueString( factorValue ) + ")" );
            }

            buf.append( '\n' );

            // Generate probe details
            for ( DifferentialExpressionAnalysisResult dear : resultSet.getResults() ) {
                StringBuilder rowBuffer = new StringBuilder();

                addGeneAnnotationsToLine( rowBuffer, dear, hasNCBIIDs, geneAnnotations );

                Map<Long, String> factorValueIdToData = new HashMap<Long, String>();
                // I don't think we can expect them in the same order.
                for ( ContrastResult contrast : dear.getContrasts() ) {
                    Double foldChange = contrast.getLogFoldChange();
                    Double pValue = contrast.getPvalue();
                    String formattedPvalue = pValue == null ? "" : String.format( DECIMAL_FORMAT, pValue );
                    String formattedFoldChange = foldChange == null ? "" : String.format( DECIMAL_FORMAT, foldChange );
                    String contrastData = "\t" + formattedFoldChange + "\t" + formattedPvalue;
                    assert contrast.getFactorValue() != null;
                    factorValueIdToData.put( contrast.getFactorValue().getId(), contrastData );
                }

                // Get them in the right order.
                for ( Long factorValueId : factorValueIdOrder ) {
                    String s = factorValueIdToData.get( factorValueId );
                    if ( s == null ) s = "";
                    rowBuffer.append( s );
                }

                buf.append( rowBuffer.toString() + '\n' );

            } // resultSet.getResults() loop
        }
        return buf.toString();
    }

    /**
     * @param rowBuffer
     * @param dear
     * @param hasNCBIIDs Whether the annotations have the NCBI gene ids. This is based on just peeking at one, so it
     *        might be wrong! But the format will be okay.
     * @param geneAnnotations
     */
    private void addGeneAnnotationsToLine( StringBuilder rowBuffer, DifferentialExpressionAnalysisResult dear,
            boolean hasNCBIIDs, Map<Long, String[]> geneAnnotations ) {
        CompositeSequence cs = dear.getProbe();
        Long csid = cs.getId();
        rowBuffer.append( cs.getName() );
        if ( geneAnnotations.containsKey( csid ) ) {
            String[] annotationStrings = geneAnnotations.get( csid );
            rowBuffer.append( "\t" + annotationStrings[1] + "\t" + annotationStrings[2] );

            // leaving out Gemma ID, which is annotationStrings[3]
            if ( hasNCBIIDs ) {
                // ncbi id.
                rowBuffer.append( "\t" + annotationStrings[4] );
            }
        } else {
            rowBuffer.append( "\t\t" );
            if ( hasNCBIIDs ) {
                rowBuffer.append( "\t" );
            }
        }
    }

    @Override
    public void deleteAllFiles( ExpressionExperiment ee ) throws IOException {
        ee = this.expressionExperimentService.thawLite( ee );

        // data files.
        deleteAndLog( getOutputFile( ee, true ) );
        deleteAndLog( getOutputFile( ee, false ) );

        // diff ex files
        Collection<DifferentialExpressionAnalysis> analyses = this.differentialExpressionAnalysisService
                .getAnalyses( ee );
        for ( DifferentialExpressionAnalysis analysis : analyses ) {
            deleteDiffExArchiveFile( analysis );
        }

        // coexpression file
        deleteAndLog( getOutputFile( getCoexpressionDataFilename( ee ) ) );

        // design file
        deleteAndLog( getOutputFile( getDesignFileName( ee, DATA_FILE_SUFFIX_COMPRESSED ) ) );
    }

    /**
     * @param analysis
     */
    @Override
    public void deleteDiffExArchiveFile( DifferentialExpressionAnalysis analysis ) throws IOException {
        String filename = getDiffExArchiveFileName( analysis );
        deleteAndLog( getOutputFile( filename ) );
    }

    @Autowired
    private DifferentialExpressionAnalyzerService analyzerService;

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDiffExpressionDataFile(ubic.gemma.model.
     * analysis
     * .expression.diff.DifferentialExpressionAnalysis, boolean)
     */
    @Override
    public File getDiffExpressionAnalysisArchiveFile( Long analysisId, boolean forceCreate ) {
        DifferentialExpressionAnalysis analysis = this.differentialExpressionAnalysisService.load( analysisId );

        String filename = getDiffExArchiveFileName( analysis );
        File f = getOutputFile( filename );

        if ( !forceCreate && f.canRead() ) {
            log.info( f + " exists, not regenerating" );
            return f;
        }

        analysis = this.differentialExpressionAnalysisService.thawFully( analysis );
        BioAssaySet experimentAnalyzed = analysis.getExperimentAnalyzed();

        /*
         * Decide if we need to extend the analysis first.
         */
        boolean extend = false;
        for ( ExpressionAnalysisResultSet rs : analysis.getResultSets() ) {
            if ( rs.getQvalueThresholdForStorage() != null && rs.getQvalueThresholdForStorage() < 1.0 ) {
                extend = true;
                break;
            }
        }

        if ( extend ) {
            log.info( "Extending an existing analysis to incldue all results" );

            ExpressionExperiment ee = experimentForBioAssaySet( experimentAnalyzed );

            Collection<ExpressionAnalysisResultSet> updatedResultSets = analyzerService.extendAnalysis( ee, analysis );

            boolean added = analysis.getResultSets().addAll( updatedResultSets );
            assert !added : "Should have simply replaced";
        }

        try {
            analysis = this.differentialExpressionAnalysisService.thawFully( analysis );
            writeDiffExArchiveFile( experimentAnalyzed, analysis );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }

        return f;
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileService#getDiffExpressionAnalysisArchiveFile(ubic.gemma.model.
     * expression.experiment.BioAssaySet, ubic.gemma.model.analysis.expression.diff.DifferentialExpressionAnalysis,
     * java.util.Collection)
     */
    @Override
    public File getDiffExpressionAnalysisArchiveFile( BioAssaySet experimentAnalyzed,
            DifferentialExpressionAnalysis analysis, Collection<ExpressionAnalysisResultSet> resultSets ) {

        assert analysis.getId() != null;

        Collection<ArrayDesign> arrayDesigns = this.expressionExperimentService
                .getArrayDesignsUsed( experimentAnalyzed );
        Map<Long, String[]> geneAnnotations = this.getGeneAnnotationsAsStrings( arrayDesigns );
        String filename = getDiffExArchiveFileName( analysis );
        File f = getOutputFile( filename );

        // We create .zip file with analysis results.
        log.info( "Creating new Differential Expression data archive file: " + f.getName() );
        try (ZipOutputStream zipOut = new ZipOutputStream( new FileOutputStream( f ) );) {

            // Add gene x factor analysis results file.
            zipOut.putNextEntry( new ZipEntry( "analysis.data.txt" ) );

            StringBuilder buf = new StringBuilder();
            buf.append( makeDiffExpressionFileHeader( analysis, resultSets, geneAnnotations ) );
            analysisResultSetsToString( resultSets, geneAnnotations, buf );
            String analysisData = buf.toString();

            zipOut.write( analysisData.getBytes() );
            zipOut.closeEntry();

            // Add a file for each result set with contrasts information.
            for ( ExpressionAnalysisResultSet resultSet : resultSets ) {
                if ( resultSet.getExperimentalFactors().size() > 1 ) {
                    continue; // Skip interactions.
                }

                String resultSetData = convertDiffExpressionResultSetData( resultSet, geneAnnotations );
                zipOut.putNextEntry( new ZipEntry( "resultset_" + resultSet.getId() + ".data.txt" ) );
                zipOut.write( resultSetData.getBytes() );
                zipOut.closeEntry();
            }

            zipOut.close();
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
        log.info( "Full results are stored in " + f.getAbsolutePath() );
        return f;
    }

    /**
     * @param experimentAnalyzed
     * @param analysis
     */
    @Override
    public void writeDiffExArchiveFile( BioAssaySet experimentAnalyzed, DifferentialExpressionAnalysis analysis )
            throws IOException {
        Collection<ArrayDesign> arrayDesigns = this.expressionExperimentService
                .getArrayDesignsUsed( experimentAnalyzed );
        Map<Long, String[]> geneAnnotations = this.getGeneAnnotationsAsStrings( arrayDesigns );
        String filename = getDiffExArchiveFileName( analysis );
        File f = getOutputFile( filename );

        // We create .zip file with analysis results.
        log.info( "Creating new Differential Expression data archive file: " + f.getName() );
        try (ZipOutputStream zipOut = new ZipOutputStream( new FileOutputStream( f ) );) {

            // Add gene x factor analysis results file.
            zipOut.putNextEntry( new ZipEntry( "analysis.data.txt" ) );
            String analysisData = convertDiffExpressionAnalysisData( analysis, geneAnnotations );
            zipOut.write( analysisData.getBytes() );
            zipOut.closeEntry();

            // Add a file for each result set with contrasts information.
            for ( ExpressionAnalysisResultSet resultSet : analysis.getResultSets() ) {
                if ( resultSet.getExperimentalFactors().size() > 1 ) {
                    continue; // Skip interactions.
                }

                assert resultSet.getQvalueThresholdForStorage() == null
                        || resultSet.getQvalueThresholdForStorage() == 1.0;

                String resultSetData = convertDiffExpressionResultSetData( resultSet, geneAnnotations );
                zipOut.putNextEntry( new ZipEntry( "resultset_" + resultSet.getId() + ".data.txt" ) );
                zipOut.write( resultSetData.getBytes() );
                zipOut.closeEntry();
            }
        }
    }

    @Override
    public File getOutputFile( ExpressionExperiment ee, boolean filtered ) {
        return getOutputFile( ee, filtered, true, false );
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#getOutputFile(ubic.gemma.model.expression.experiment.
     * ExpressionExperiment, boolean)
     */
    @Override
    public File getOutputFile( ExpressionExperiment ee, boolean filtered, boolean compressed, boolean temporary ) {
        String filteredAdd = "";
        if ( !filtered ) {
            filteredAdd = ".unfilt";
        }
        String suffix;

        if ( compressed ) {
            suffix = DATA_FILE_SUFFIX_COMPRESSED;
        } else {
            suffix = DATA_FILE_SUFFIX;
        }

        String filename = getDataFileName( ee, filteredAdd, suffix );

        // randomize file name if temporary in case of access by more than one user at once
        if ( temporary ) {

            filename = RandomStringUtils.randomAlphabetic( 6 ) + filename;

        }

        return getOutputFile( filename, temporary );
    }

    @Override
    public File getOutputFile( String filename ) {
        return getOutputFile( filename, false );

    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#getOutputFile(java.lang.String)
     */
    @Override
    public File getOutputFile( String filename, boolean temporary ) {
        String fullFilePath;
        if ( temporary ) {
            fullFilePath = TMP_DATA_DIR + filename;
        } else {
            fullFilePath = DATA_DIR + filename;
        }
        File f = new File( fullFilePath );

        if ( f.exists() ) {
            return f;
        }

        File parentDir = f.getParentFile();
        if ( !parentDir.exists() ) parentDir.mkdirs();
        return f;
    }

    @Override
    public File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName, boolean compress )
            throws IOException {
        File f = new File( fileName );
        return writeDataFile( ee, filtered, f, compress );
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateCoexpressionDataFile(ubic.gemma.model.
     * expression
     * .experiment.ExpressionExperiment, boolean)
     */
    @Override
    public File writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean forceWrite ) {

        ee = expressionExperimentService.thawLite( ee );

        String getCoexpressionDataFilename = getCoexpressionDataFilename( ee );
        String filename = getCoexpressionDataFilename;
        try {
            File f = getOutputFile( filename );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new Co-Expression data file: " + f.getName() );
            writeCoexpressionData( f, ee );
            return f;
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDataFile(ubic.gemma.model.expression.
     * experiment
     * .ExpressionExperiment, boolean, boolean)
     */
    @Override
    public File writeOrLocateDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) {

        try {
            File f = getOutputFile( ee, filtered );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }
            return writeDataFile( ee, filtered, f, true );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    @Override
    public File writeTemporaryDataFile( ExpressionExperiment ee, boolean filtered ) {

        try {

            File f = getOutputFile( ee, filtered, false, true );

            return writeDataFile( ee, filtered, f, false );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDataFile(ubic.gemma.model.common.
     * quantitationtype
     * .QuantitationType, boolean)
     */
    @Override
    public File writeOrLocateDataFile( QuantitationType type, boolean forceWrite ) {

        try {
            File f = getOutputFile( type );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new quantitation type expression data file: " + f.getName() );

            Collection<? extends DesignElementDataVector> vectors = designElementDataVectorService.find( type );
            Collection<ArrayDesign> arrayDesigns = getArrayDesigns( vectors );
            Map<CompositeSequence, String[]> geneAnnotations = this.getGeneAnnotationsAsStringsByProbe( arrayDesigns );

            if ( vectors.size() == 0 ) {
                log.warn( "No vectors for " + type );
                return null;
            }

            writeVectors( f, type.getRepresentation(), vectors, geneAnnotations );
            return f;
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDesignFile(ubic.gemma.model.expression.
     * experiment
     * .ExpressionExperiment, boolean)
     */
    @Override
    public File writeOrLocateDesignFile( ExpressionExperiment ee, boolean forceWrite ) {

        ee = expressionExperimentService.thawLite( ee );

        String filename = getDesignFileName( ee, DATA_FILE_SUFFIX_COMPRESSED );
        try {
            File f = getOutputFile( filename );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new experimental design file: " + f.getName() );
            return writeDesignMatrix( f, ee, true );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileService#writeTemporaryDesignFile(ubic.gemma.model.expression.
     * experiment
     * .ExpressionExperiment)
     */
    @Override
    public File writeTemporaryDesignFile( ExpressionExperiment ee ) {

        ee = expressionExperimentService.thawLite( ee );

        // not compressed
        String filename = getDesignFileName( ee, DATA_FILE_SUFFIX );

        filename = RandomStringUtils.randomAlphabetic( 6 ) + filename;
        try {
            File f = getOutputFile( filename, true );

            log.info( "Creating new experimental design file: " + f.getName() );
            return writeDesignMatrix( f, ee, true, false );
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateDiffExpressionDataFiles(ubic.gemma.model.
     * expression.experiment.ExpressionExperiment, boolean)
     */
    @Override
    public Collection<File> writeOrLocateDiffExpressionDataFiles( ExpressionExperiment ee, boolean forceWrite ) {

        ee = this.expressionExperimentService.thawLite( ee );

        Collection<DifferentialExpressionAnalysis> analyses = this.differentialExpressionAnalysisService
                .getAnalyses( ee );

        Collection<File> result = new HashSet<>();
        for ( DifferentialExpressionAnalysis analysis : analyses ) {
            assert analysis.getId() != null;
            result.add( this.getDiffExpressionAnalysisArchiveFile( analysis.getId(), forceWrite ) );
        }

        return result;

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateJSONDataFile(ubic.gemma.model.expression.
     * experiment.ExpressionExperiment, boolean, boolean)
     */
    @Override
    public File writeOrLocateJSONDataFile( ExpressionExperiment ee, boolean forceWrite, boolean filtered ) {

        try {
            File f = getOutputFile( ee, filtered );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new JSON expression data file: " + f.getName() );
            ExpressionDataDoubleMatrix matrix = getDataMatrix( ee, filtered, f );

            Collection<ArrayDesign> arrayDesigns = expressionExperimentService.getArrayDesignsUsed( ee );
            Map<Long, Collection<Gene>> geneAnnotations = this.getGeneAnnotations( arrayDesigns );

            writeJson( f, geneAnnotations, matrix );
            return f;
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.core.analysis.service.ExpressionDataFileSerivce#writeOrLocateJSONDataFile(ubic.gemma.model.common.
     * quantitationtype.QuantitationType, boolean)
     */
    @Override
    public File writeOrLocateJSONDataFile( QuantitationType type, boolean forceWrite ) {

        try {
            File f = getJSONOutputFile( type );
            if ( !forceWrite && f.canRead() ) {
                log.info( f + " exists, not regenerating" );
                return f;
            }

            log.info( "Creating new quantitation type  JSON data file: " + f.getName() );

            Collection<? extends DesignElementDataVector> vectors = designElementDataVectorService.find( type );

            if ( vectors.size() == 0 ) {
                log.warn( "No vectors for " + type );
                return null;
            }

            writeJson( f, type.getRepresentation(), vectors );
            return f;
        } catch ( IOException e ) {
            throw new RuntimeException( e );
        }
    }

    /**
     * Given diff exp analysis and gene annotation generate header and tab delimited data. The output is qValue....
     * 
     * @param analysis
     * @param geneAnnotations
     * @return
     */
    private String convertDiffExpressionAnalysisData( DifferentialExpressionAnalysis analysis,
            Map<Long, String[]> geneAnnotations ) {
        Collection<ExpressionAnalysisResultSet> results = analysis.getResultSets();
        if ( results == null || results.isEmpty() ) {
            log.warn( "No differential expression results found for " + analysis );
            return "";
        }

        StringBuilder buf = new StringBuilder();

        buf.append( makeDiffExpressionFileHeader( analysis, analysis.getResultSets(), geneAnnotations ) );
        analysisResultSetsToString( results, geneAnnotations, buf );

        return buf.toString();
    }

    /**
     * Given result set and gene annotation generate header and tab delimited data. The output is foldChange and pValue
     * associated with each contrast.
     * 
     * @param resultSet
     * @param geneAnnotations
     * @return
     */
    private String convertDiffExpressionResultSetData( ExpressionAnalysisResultSet resultSet,
            Map<Long, String[]> geneAnnotations ) {
        StringBuilder buf = new StringBuilder();
        // Write header.
        buf.append( makeDiffExpressionResultSetFileHeader( resultSet, geneAnnotations ) );
        // Write contrasts data.
        buf.append( analysisResultSetWithContrastsToString( resultSet, geneAnnotations ) );

        return buf.toString();
    }

    /**
     * @param f1
     */
    private void deleteAndLog( File f1 ) {
        if ( f1.canWrite() && f1.delete() ) {
            log.info( "Deleted: " + f1 );
        }
    }

    /**
     * @param vectors
     * @return
     */
    private Collection<ArrayDesign> getArrayDesigns( Collection<? extends DesignElementDataVector> vectors ) {
        Collection<ArrayDesign> ads = new HashSet<ArrayDesign>();
        for ( DesignElementDataVector v : vectors ) {
            ads.add( v.getDesignElement().getArrayDesign() );
        }
        return ads;
    }

    /**
     * @param ee
     * @return
     */
    private String getCoexpressionDataFilename( ExpressionExperiment ee ) {
        return ee.getId() + "_" + FileTools.cleanForFileName( ee.getShortName() ) + "_coExp"
                + DATA_FILE_SUFFIX_COMPRESSED;
    }

    /**
     * @param ee
     * @param filteredAdd
     * @return Name, without full path.
     */
    private String getDataFileName( ExpressionExperiment ee, String filteredAdd, String suffix ) {
        return ee.getId() + "_" + FileTools.cleanForFileName( ee.getShortName() ) + "_expmat" + filteredAdd + suffix;
    }

    /**
     * @param ee
     * @param filtered
     * @param f
     * @return
     */
    private ExpressionDataDoubleMatrix getDataMatrix( ExpressionExperiment ee, boolean filtered, File f ) {

        FilterConfig filterConfig = new FilterConfig();
        filterConfig.setIgnoreMinimumSampleThreshold( true );
        filterConfig.setIgnoreMinimumRowsThreshold( true );
        ee = expressionExperimentService.thawLite( ee );
        ExpressionDataDoubleMatrix matrix;
        if ( filtered ) {
            matrix = expressionDataMatrixService.getFilteredMatrix( ee, filterConfig );
        } else {
            matrix = expressionDataMatrixService.getProcessedExpressionDataMatrix( ee );
        }
        return matrix;
    }

    /**
     * @param ee
     * @return
     */
    private String getDesignFileName( ExpressionExperiment ee, String suffix ) {
        return ee.getId() + "_" + FileTools.cleanForFileName( ee.getShortName() ) + "_expdesign" + suffix;
    }

    /**
     * @param diff
     * @return
     */
    private String getDiffExArchiveFileName( DifferentialExpressionAnalysis diff ) {
        BioAssaySet experimentAnalyzed = diff.getExperimentAnalyzed();

        ExpressionExperiment ee;
        if ( experimentAnalyzed instanceof ExpressionExperiment ) {
            ee = ( ExpressionExperiment ) experimentAnalyzed;
        } else if ( experimentAnalyzed instanceof ExpressionExperimentSubSet ) {
            ExpressionExperimentSubSet subset = ( ExpressionExperimentSubSet ) experimentAnalyzed;
            ee = subset.getSourceExperiment();
        } else {
            throw new UnsupportedOperationException( "Don't know about " + experimentAnalyzed.getClass().getName() );
        }

        return experimentAnalyzed.getId() + "_" + FileTools.cleanForFileName( ee.getShortName() ) + "_diffExpAnalysis_"
                + diff.getId() + DATA_ARCHIVE_FILE_SUFFIX;
    }

    /**
     * @param fv
     * @return
     */
    private String getFactorValueString( FactorValue fv ) {
        if ( fv == null ) return "null";

        if ( fv.getCharacteristics() != null && fv.getCharacteristics().size() > 0 ) {
            String fvString = "";
            for ( Characteristic c : fv.getCharacteristics() ) {
                fvString += c.getValue() + " ";
            }
            return fvString;
        } else if ( fv.getMeasurement() != null ) {
            return fv.getMeasurement().getValue();
        } else if ( fv.getValue() != null && !fv.getValue().isEmpty() ) {
            return fv.getValue();
        } else
            return "absent ";
    }

    /**
     * @param ads
     * @return
     */
    private Map<Long, Collection<Gene>> getGeneAnnotations( Collection<ArrayDesign> ads ) {
        Map<Long, Collection<Gene>> annots = new HashMap<Long, Collection<Gene>>();
        for ( ArrayDesign arrayDesign : ads ) {
            arrayDesign = arrayDesignService.thaw( arrayDesign );
            annots.putAll( ArrayDesignAnnotationServiceImpl.readAnnotationFile( arrayDesign ) );
        }
        return annots;
    }

    /**
     * @param ads
     * @return Map of composite sequence ids to an array of strings: [probe name, genes symbol(s), gene Name(s), gemma
     *         id(s), ncbi id(s)].
     */
    private Map<Long, String[]> getGeneAnnotationsAsStrings( Collection<ArrayDesign> ads ) {
        Map<Long, String[]> annots = new HashMap<Long, String[]>();
        for ( ArrayDesign arrayDesign : ads ) {
            arrayDesign = arrayDesignService.thaw( arrayDesign );
            annots.putAll( ArrayDesignAnnotationServiceImpl.readAnnotationFileAsString( arrayDesign ) );
        }
        return annots;
    }

    /**
     * @param ads
     * @return
     */
    private Map<CompositeSequence, String[]> getGeneAnnotationsAsStringsByProbe( Collection<ArrayDesign> ads ) {
        Map<CompositeSequence, String[]> annots = new HashMap<CompositeSequence, String[]>();
        for ( ArrayDesign arrayDesign : ads ) {
            arrayDesign = arrayDesignService.thaw( arrayDesign );

            Map<Long, CompositeSequence> csidmap = EntityUtils.getIdMap( arrayDesign.getCompositeSequences() );

            Map<Long, String[]> geneAnnots = ArrayDesignAnnotationServiceImpl.readAnnotationFileAsString( arrayDesign );

            for ( Entry<Long, String[]> e : geneAnnots.entrySet() ) {

                if ( !csidmap.containsKey( e.getKey() ) ) {
                    continue;
                }

                annots.put( csidmap.get( e.getKey() ), e.getValue() );

            }

        }
        return annots;
    }

    /**
     * @param type
     * @return
     * @throws IOException
     */
    private File getJSONOutputFile( QuantitationType type ) throws IOException {
        String filename = getJSONOutputFilename( type );
        String fullFilePath = DATA_DIR + filename;

        File f = new File( fullFilePath );

        if ( f.exists() ) {
            log.warn( "Will overwrite existing file " + f );
            f.delete();
        }

        File parentDir = f.getParentFile();
        if ( !parentDir.exists() ) parentDir.mkdirs();
        f.createNewFile();
        return f;
    }

    /**
     * @param type
     * @return Name, without full path.
     */
    private String getJSONOutputFilename( QuantitationType type ) {
        return FileTools.cleanForFileName( type.getName() ) + JSON_FILE_SUFFIX;
    }

    private File getOutputFile( QuantitationType type ) {
        String filename = getOutputFilename( type );
        return getOutputFile( filename );
    }

    /**
     * @param type
     * @return Name, without full path.
     */
    private String getOutputFilename( QuantitationType type ) {
        return type.getId() + "_" + FileTools.cleanForFileName( type.getName() ) + DATA_FILE_SUFFIX_COMPRESSED;
    }

    /**
     * @param analysis
     * @param geneAnnotations
     * @param buf
     * @return header string
     */
    private String makeDiffExpressionFileHeader( DifferentialExpressionAnalysis analysis,
            Collection<ExpressionAnalysisResultSet> resultSets, Map<Long, String[]> geneAnnotations ) {

        if ( analysis.getId() != null ) // might be if we are using -nodb
            differentialExpressionAnalysisService.thaw( analysis ); // bug 4023

        StringBuilder buf = new StringBuilder();

        BioAssaySet bas = analysis.getExperimentAnalyzed();

{'fraction_non_alphanumeric': 0.0954872542564199, 'fraction_numerical': 0.0013639356598626657, 'mean_word_length': 2.3931378416118094, 'pattern_counts': {'":': 4, '<': 129, '<?xml version=': 0, '>': 137, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6082645', 'n_tokens_mistral': 22678, 'n_tokens_neox': 21776, 'n_words': 5209}