<commit_before>/*
 * The Gemma project
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.datastructure.matrix;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.lang.RandomStringUtils;

import ubic.basecode.io.ByteArrayConverter;
import ubic.gemma.model.common.quantitationtype.QuantitationType;
import ubic.gemma.model.expression.bioAssay.BioAssay;
import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.expression.designElement.DesignElement;
import ubic.gemma.model.expression.designElement.Reporter;
import ubic.gemma.testing.TestPersistentObjectHelper;
import junit.framework.TestCase;

/**
 * @author pavlidis
 * @version $Id$
 */
public class MatrixConversionTest extends TestCase {

    private static final int NUM_BIOMATERIALS = 40;
    private static final int NUM_CS = 200;

    public final void testColumnMapping() {
        Collection<QuantitationType> quantTypes = new HashSet<QuantitationType>();
        for ( int quantitationTypeNum = 0; quantitationTypeNum < 2; quantitationTypeNum++ ) {
            QuantitationType quantType = TestPersistentObjectHelper.getTestNonPersistentQuantitationType();
            quantType.setId( ( long ) quantitationTypeNum );
            quantTypes.add( quantType );
        }

        Collection<DesignElementDataVector> vectors = getDesignElementDataVectors( quantTypes );
        ExpressionDataDoubleMatrix mat = new ExpressionDataDoubleMatrix( vectors, quantTypes.iterator().next() );
        assertEquals( NUM_CS, mat.rows() );
        assertEquals( NUM_BIOMATERIALS, mat.columns() );

        for ( DesignElementDataVector vector : vectors ) {
            DesignElement de = vector.getDesignElement();
            Double[] res = mat.getRow( de );
            assertEquals( NUM_BIOMATERIALS, res.length );
            // System.err.print( de.getName() );
            for ( int i = 0; i < res.length; i++ ) {
                Double r = res[i];
                assertTrue( "Expected " + i + ", got " + r, i == r.intValue() || r.equals( Double.NaN ) );
                // System.err.print( "\t" + r );
            }
            // System.err.print( "\n" );
        }

    }

    /**
     * Creates an ugly (but not unusual) situation where there are two bioassay dimensions with different sizes,
     * referring to the same set of biomaterials.
     * 
     * @return
     */
    public Collection<DesignElementDataVector> getDesignElementDataVectors( Collection<QuantitationType> quantTypes ) {
        Collection<DesignElementDataVector> vectors = new HashSet<DesignElementDataVector>();

        List<CompositeSequence> sequences = getCompositeSequences();

        List<BioMaterial> bioMaterials = getBioMaterials(); // resused

        for ( QuantitationType quantType : quantTypes ) {

            /*
             * Create two bioassay dimension which overlap; "A" does not use all the biomaterials.
             */
            BioAssayDimension baDimA = BioAssayDimension.Factory.newInstance();
            Iterator<BioMaterial> bmita = bioMaterials.iterator();
            for ( long i = 0; i < NUM_BIOMATERIALS - 20; i++ ) {
                BioAssay ba = ubic.gemma.model.expression.bioAssay.BioAssay.Factory.newInstance();
                ba.setName( RandomStringUtils.randomNumeric( 5 ) + "_testbioassay" );
                ba.getSamplesUsed().add( bmita.next() );
                ba.setId( i );
                baDimA.getBioAssays().add( ba );
            }

            BioAssayDimension baDimB = BioAssayDimension.Factory.newInstance();
            Iterator<BioMaterial> bmitb = bioMaterials.iterator();
            for ( long i = 0; i < NUM_BIOMATERIALS; i++ ) {
                BioAssay ba = ubic.gemma.model.expression.bioAssay.BioAssay.Factory.newInstance();
                ba.setName( RandomStringUtils.randomNumeric( 5 ) + "_testbioassay" );
                ba.getSamplesUsed().add( bmitb.next() );
                ba.setId( i );
                baDimB.getBioAssays().add( ba );
            }

            // bio.a gets cs 0-99, bio.b gets 100-199.
            long j = 0;
            for ( ; j < NUM_CS - 100; j++ ) {
                DesignElementDataVector vector = DesignElementDataVector.Factory.newInstance();
                double[] data = new double[baDimA.getBioAssays().size()];
                for ( int k = 0; k < data.length; k++ ) {
                    data[k] = ( double ) k;<commit_msg>set bioassaydim names (required for hashCode)<commit_after>/*
 * The Gemma project
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.datastructure.matrix;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.lang.RandomStringUtils;

import ubic.basecode.io.ByteArrayConverter;
import ubic.gemma.model.common.quantitationtype.QuantitationType;
import ubic.gemma.model.expression.bioAssay.BioAssay;
import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
import ubic.gemma.model.expression.biomaterial.BioMaterial;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.expression.designElement.DesignElement;
import ubic.gemma.model.expression.designElement.Reporter;
import ubic.gemma.testing.TestPersistentObjectHelper;
import junit.framework.TestCase;

/**
 * @author pavlidis
 * @version $Id$
 */
public class MatrixConversionTest extends TestCase {

    private static final int NUM_BIOMATERIALS = 40;
    private static final int NUM_CS = 200;

    public final void testColumnMapping() {
        Collection<QuantitationType> quantTypes = new HashSet<QuantitationType>();
        for ( int quantitationTypeNum = 0; quantitationTypeNum < 2; quantitationTypeNum++ ) {
            QuantitationType quantType = TestPersistentObjectHelper.getTestNonPersistentQuantitationType();
            quantType.setId( ( long ) quantitationTypeNum );
            quantTypes.add( quantType );
        }

        Collection<DesignElementDataVector> vectors = getDesignElementDataVectors( quantTypes );
        ExpressionDataDoubleMatrix mat = new ExpressionDataDoubleMatrix( vectors, quantTypes.iterator().next() );
        assertEquals( NUM_CS, mat.rows() );
        assertEquals( NUM_BIOMATERIALS, mat.columns() );

        for ( DesignElementDataVector vector : vectors ) {
            DesignElement de = vector.getDesignElement();
            Double[] res = mat.getRow( de );
            assertEquals( NUM_BIOMATERIALS, res.length );
            // System.err.print( de.getName() );
            for ( int i = 0; i < res.length; i++ ) {
                Double r = res[i];
                assertTrue( "Expected " + i + ", got " + r, i == r.intValue() || r.equals( Double.NaN ) );
                // System.err.print( "\t" + r );
            }
            // System.err.print( "\n" );
        }

    }

    /**
     * Creates an ugly (but not unusual) situation where there are two bioassay dimensions with different sizes,
     * referring to the same set of biomaterials.
     * 
     * @return
     */
    public Collection<DesignElementDataVector> getDesignElementDataVectors( Collection<QuantitationType> quantTypes ) {
        Collection<DesignElementDataVector> vectors = new HashSet<DesignElementDataVector>();

        List<CompositeSequence> sequences = getCompositeSequences();

        List<BioMaterial> bioMaterials = getBioMaterials(); // resused

        for ( QuantitationType quantType : quantTypes ) {

            /*
             * Create two bioassay dimension which overlap; "A" does not use all the biomaterials.
             */
            BioAssayDimension baDimA = BioAssayDimension.Factory.newInstance();
            Iterator<BioMaterial> bmita = bioMaterials.iterator();
            for ( long i = 0; i < NUM_BIOMATERIALS - 20; i++ ) {
                BioAssay ba = ubic.gemma.model.expression.bioAssay.BioAssay.Factory.newInstance();
                ba.setName( RandomStringUtils.randomNumeric( 5 ) + "_testbioassay" );
                ba.getSamplesUsed().add( bmita.next() );
                ba.setId( i );
                baDimA.getBioAssays().add( ba );
            }
            baDimA.setName( RandomStringUtils.randomAlphanumeric( 10 ) );

            BioAssayDimension baDimB = BioAssayDimension.Factory.newInstance();
            Iterator<BioMaterial> bmitb = bioMaterials.iterator();
            for ( long i = 0; i < NUM_BIOMATERIALS; i++ ) {
                BioAssay ba = ubic.gemma.model.expression.bioAssay.BioAssay.Factory.newInstance();
                ba.setName( RandomStringUtils.randomNumeric( 5 ) + "_testbioassay" );
                ba.getSamplesUsed().add( bmitb.next() );
                ba.setId( i );
                baDimB.getBioAssays().add( ba );
            }
            baDimB.setName( RandomStringUtils.randomAlphanumeric( 10 ) );

            // bio.a gets cs 0-99, bio.b gets 100-199.
            long j = 0;
            for ( ; j < NUM_CS - 100; j++ ) {
                DesignElementDataVector vector = DesignElementDataVector.Factory.newInstance();
                double[] data = new double[baDimA.getBioAssays().size()];
                for ( int k = 0; k < data.length; k++ ) {
                    data[k] = ( double ) k;
{'fraction_non_alphanumeric': 0.10260676650027732, 'fraction_numerical': 0.007025328156775744, 'mean_word_length': 2.692491467576792, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2896591', 'n_tokens_mistral': 3126, 'n_tokens_neox': 2944, 'n_words': 744}