<commit_before>/*
 * The Gemma project
 * 
 * Copyright (c) 2007 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.datastructure.matrix;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import ubic.basecode.dataStructure.matrix.DoubleMatrix;
import ubic.basecode.math.MatrixStats;
import ubic.gemma.analysis.preprocess.UnknownLogScaleException;
import ubic.gemma.analysis.preprocess.filter.ExpressionExperimentFilter;
import ubic.gemma.model.common.quantitationtype.QuantitationType;
import ubic.gemma.model.common.quantitationtype.ScaleType;
import ubic.gemma.model.expression.bioAssay.BioAssay;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import cern.colt.matrix.DoubleMatrix1D;

/**
 * Perform various computations on ExpressionDataMatrices (usually in-place).
 * 
 * @author pavlidis
 * @version $Id$
 */
public class ExpressionDataDoubleMatrixUtil {

    private static final double LOGARITHM_BASE = 2.0;

    private static Log log = LogFactory.getLog( ExpressionDataDoubleMatrixUtil.class.getName() );

    /**
     * Log transform if necessary, do any required filtering prior to analysis.
     * 
     * @param quantitationType
     * @param dmatrix
     */
    public static ExpressionDataDoubleMatrix filterAndLogTransform( QuantitationType quantitationType,
            ExpressionDataDoubleMatrix dmatrix ) {

        ScaleType scaleType = findScale( quantitationType, dmatrix.getMatrix() );

        if ( scaleType.equals( ScaleType.LOG2 ) ) {
            log.info( "Data is already on a log2 scale" );
        } else if ( scaleType.equals( ScaleType.LN ) ) {
            log.info( " **** Converting from ln to log2 **** " );
            MatrixStats.convertToLog2( dmatrix.getMatrix(), Math.E );
        } else if ( scaleType.equals( ScaleType.LOG10 ) ) {
            log.info( " **** Converting from log10 to log2 **** " );
            MatrixStats.convertToLog2( dmatrix.getMatrix(), 10 );
        } else if ( scaleType.equals( ScaleType.LINEAR ) ) {
            log.info( " **** LOG TRANSFORMING **** " );
            MatrixStats.logTransform( dmatrix.getMatrix() );
        } else if ( scaleType.equals( ScaleType.COUNT ) ) {
            log.info( " **** Converting from count to log2 counts per million **** " );
            DoubleMatrix1D librarySize = MatrixStats.colSums( dmatrix.getMatrix() );
            MatrixStats.convertToLog2Cpm( dmatrix.getMatrix(), librarySize );
        } else {
            throw new UnknownLogScaleException( "Can't figure out what scale the data are on" );
        }

        /*
         * We do this second because doing it first causes some kind of subtle problem ... (round off? I could not
         * really track this down).
         * 
         * Remove zero-variance rows, but also rows that have lots of equal values even if variance is non-zero. This
         * happens when data is "clipped" (e.g., all values under 10 set to 10).
         */
        int r = dmatrix.rows();
        dmatrix = ExpressionExperimentFilter.zeroVarianceFilter( dmatrix );
        if ( dmatrix.rows() < r ) {
            log.info( ( r - dmatrix.rows() ) + " rows removed due to low variance" );
        }
        r = dmatrix.rows();
        /* FIXME refactor as constant. Idea is to skip this if the data set is really small, but this is totally ad hoc. */
        if ( dmatrix.columns() > 4 ) {
            dmatrix = ExpressionExperimentFilter.tooFewDistinctValues( dmatrix, 0.5 );
            if ( dmatrix.rows() < r ) {
                log.info( ( r - dmatrix.rows() ) + " rows removed due to too many identical values" );
            }
        }

        return dmatrix;

    }

    /**
     * @param quantitationType
     * @param namedMatrix
     * @return
     * @see ExpressionExperimentFilter for a related implementation.
     */
    public static ScaleType findScale( QuantitationType quantitationType,
            DoubleMatrix<CompositeSequence, BioMaterial> namedMatrix ) {

        if ( quantitationType.getScale() != null ) {
            if ( quantitationType.getScale().equals( ScaleType.LOG2 ) ) {
                return ScaleType.LOG2;
            } else if ( quantitationType.getScale().equals( ScaleType.LOG10 ) ) {
                return ScaleType.LOG10;
            } else if ( quantitationType.getScale().equals( ScaleType.LN ) ) {
                return ScaleType.LN;
            } else if ( quantitationType.getScale().equals( ScaleType.COUNT ) ) {
                return ScaleType.COUNT;
            } else if ( quantitationType.getScale().equals( ScaleType.LOGBASEUNKNOWN ) ) {
                throw new UnknownLogScaleException(
                        "Sorry, data on an unknown log scale is not supported. Please check the quantitation types, "
                                + "and make sure the data is expressed in terms of log2 or un-logged data  ("
                                + quantitationType + ")" );
            }
        }

        for ( int i = 0; i < namedMatrix.rows(); i++ ) {
            for ( int j = 0; j < namedMatrix.columns(); j++ ) {
                double v = namedMatrix.get( i, j );
                if ( v > 20 ) {
                    log.debug( "Data has large values, doesn't look log transformed" );
                    return ScaleType.LINEAR;
                }
            }
        }

        throw new UnknownLogScaleException( "Data look log tranformed, not sure about base (" + quantitationType
                + ")" );<commit_msg>to help figure out 4011<commit_after>/*
 * The Gemma project
 * 
 * Copyright (c) 2007 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.datastructure.matrix;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import ubic.basecode.dataStructure.matrix.DoubleMatrix;
import ubic.basecode.math.MatrixStats;
import ubic.gemma.analysis.preprocess.UnknownLogScaleException;
import ubic.gemma.analysis.preprocess.filter.ExpressionExperimentFilter;
import ubic.gemma.model.common.quantitationtype.QuantitationType;
import ubic.gemma.model.common.quantitationtype.ScaleType;
import ubic.gemma.model.expression.bioAssay.BioAssay;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import cern.colt.matrix.DoubleMatrix1D;

/**
 * Perform various computations on ExpressionDataMatrices (usually in-place).
 * 
 * @author pavlidis
 * @version $Id$
 */
public class ExpressionDataDoubleMatrixUtil {

    /**
     * 
     */
    private static final int LARGEST_EXPECTED_LOGGED_VALUE = 20;

    private static final double LOGARITHM_BASE = 2.0;

    private static Log log = LogFactory.getLog( ExpressionDataDoubleMatrixUtil.class.getName() );

    /**
     * Log transform if necessary, do any required filtering prior to analysis.
     * 
     * @param quantitationType
     * @param dmatrix
     */
    public static ExpressionDataDoubleMatrix filterAndLogTransform( QuantitationType quantitationType,
            ExpressionDataDoubleMatrix dmatrix ) {

        ScaleType scaleType = findScale( quantitationType, dmatrix.getMatrix() );

        if ( scaleType.equals( ScaleType.LOG2 ) ) {
            log.info( "Data is already on a log2 scale" );
        } else if ( scaleType.equals( ScaleType.LN ) ) {
            log.info( " **** Converting from ln to log2 **** " );
            MatrixStats.convertToLog2( dmatrix.getMatrix(), Math.E );
        } else if ( scaleType.equals( ScaleType.LOG10 ) ) {
            log.info( " **** Converting from log10 to log2 **** " );
            MatrixStats.convertToLog2( dmatrix.getMatrix(), 10 );
        } else if ( scaleType.equals( ScaleType.LINEAR ) ) {
            log.info( " **** LOG TRANSFORMING **** " );
            MatrixStats.logTransform( dmatrix.getMatrix() );
        } else if ( scaleType.equals( ScaleType.COUNT ) ) {
            log.info( " **** Converting from count to log2 counts per million **** " );
            DoubleMatrix1D librarySize = MatrixStats.colSums( dmatrix.getMatrix() );
            MatrixStats.convertToLog2Cpm( dmatrix.getMatrix(), librarySize );
        } else {
            throw new UnknownLogScaleException( "Can't figure out what scale the data are on" );
        }

        /*
         * We do this second because doing it first causes some kind of subtle problem ... (round off? I could not
         * really track this down).
         * 
         * Remove zero-variance rows, but also rows that have lots of equal values even if variance is non-zero. This
         * happens when data is "clipped" (e.g., all values under 10 set to 10).
         */
        int r = dmatrix.rows();
        dmatrix = ExpressionExperimentFilter.zeroVarianceFilter( dmatrix );
        if ( dmatrix.rows() < r ) {
            log.info( ( r - dmatrix.rows() ) + " rows removed due to low variance" );
        }
        r = dmatrix.rows();
        /* FIXME refactor as constant. Idea is to skip this if the data set is really small, but this is totally ad hoc. */
        if ( dmatrix.columns() > 4 ) {
            dmatrix = ExpressionExperimentFilter.tooFewDistinctValues( dmatrix, 0.5 );
            if ( dmatrix.rows() < r ) {
                log.info( ( r - dmatrix.rows() ) + " rows removed due to too many identical values" );
            }
        }

        return dmatrix;

    }

    /**
     * @param quantitationType
     * @param namedMatrix
     * @return ScaleType
     * @see ExpressionExperimentFilter for a related implementation.
     */
    public static ScaleType findScale( QuantitationType quantitationType,
            DoubleMatrix<CompositeSequence, BioMaterial> namedMatrix ) {

        if ( quantitationType.getScale() != null ) {
            if ( quantitationType.getScale().equals( ScaleType.LOG2 ) ) {
                return ScaleType.LOG2;
            } else if ( quantitationType.getScale().equals( ScaleType.LOG10 ) ) {
                return ScaleType.LOG10;
            } else if ( quantitationType.getScale().equals( ScaleType.LN ) ) {
                return ScaleType.LN;
            } else if ( quantitationType.getScale().equals( ScaleType.COUNT ) ) {
                return ScaleType.COUNT;
            } else if ( quantitationType.getScale().equals( ScaleType.LOGBASEUNKNOWN ) ) {
                throw new UnknownLogScaleException(
                        "Sorry, data on an unknown log scale is not supported. Please check the quantitation types, "
                                + "and make sure the data is expressed in terms of log2 or un-logged data  ("
                                + quantitationType + ")" );
            }
        }

        if ( namedMatrix.rows() == 0 || namedMatrix.columns() == 0 ) {
            throw new UnknownLogScaleException( "Cannot figure out scale without data (" + quantitationType + ")" );
        }

        for ( int i = 0; i < namedMatrix.rows(); i++ ) {
            for ( int j = 0; j < namedMatrix.columns(); j++ ) {
                double v = namedMatrix.get( i, j );
                if ( v > LARGEST_EXPECTED_LOGGED_VALUE ) {
                    log.debug( "Data has large values, doesn't look log transformed" );
                    return ScaleType.LINEAR;
                }
            }
        }

        throw new UnknownLogScaleException( "Data look log tranformed, not sure about base (" + quantitationType + ")" );
{'fraction_non_alphanumeric': 0.0983619592875318, 'fraction_numerical': 0.007474554707379135, 'mean_word_length': 2.3809139784946236, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5222042', 'n_tokens_mistral': 3497, 'n_tokens_neox': 3309, 'n_words': 1017}