<commit_before>/*
 * Copyright 2008-2010 Microarray Informatics Team, EMBL-European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *
 * For further details of the Gene Expression Atlas project, including source code,
 * downloads and documentation, please see:
 *
 * http://gxa.github.com/gxa
 */

package uk.ac.ebi.gxa.data;

import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.primitives.Floats;
import uk.ac.ebi.gxa.utils.CollectionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.ac.ebi.microarray.atlas.model.*;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.*;

public class ExperimentWithData {
    private final Logger log = LoggerFactory.getLogger(getClass());

    private final AtlasDataDAO atlasDataDAO;
    private final Experiment experiment;

    private final Map<ArrayDesign,DataProxy> proxies = new HashMap<ArrayDesign,DataProxy>();

    // cached data
    private final Map<ArrayDesign, String[]> designElementAccessions = new HashMap<ArrayDesign, String[]>();

    ExperimentWithData(@Nonnull AtlasDataDAO atlasDataDAO, @Nonnull Experiment experiment) {
        this.atlasDataDAO = atlasDataDAO;
        this.experiment = experiment;
    }

    public Experiment getExperiment() {
        return experiment;
    }

    /**
     * @param criteria the criteria to choose arrayDesign
     * @return first arrayDesign used in experiment, that matches criteria;
     *         or null if no arrayDesign has been found
     */
    public ArrayDesign findArrayDesign(Predicate<ArrayDesign> criteria) {
        for (ArrayDesign ad : experiment.getArrayDesigns()) {
            if (criteria.apply(ad)) {
                return ad;
            }
        }
        return null;
    }

    AtlasDataDAO getDataDAO() {
        return atlasDataDAO;
    }

    DataProxy getProxy(ArrayDesign arrayDesign) throws AtlasDataException {
        DataProxy p = proxies.get(arrayDesign);
        if (p == null) {
            p = atlasDataDAO.createDataProxy(experiment, arrayDesign);
            proxies.put(arrayDesign, p);
        }
        return p;
    }

    public NetCDFDataCreator getDataCreator(ArrayDesign arrayDesign) {
        return new NetCDFDataCreator(atlasDataDAO, experiment, arrayDesign);
    }

    public void updateAllData() throws AtlasDataException {
        final DataUpdater updater = new DataUpdater();
        for (ArrayDesign arrayDesign : experiment.getArrayDesigns()) {
            updater.update(arrayDesign);
        }
    }

    public void updateDataToNewestVersion() throws AtlasDataException {
        for (ArrayDesign arrayDesign : experiment.getArrayDesigns()) {
            final DataProxy proxy = getProxy(arrayDesign);
            if ("2.0".equals(proxy.getVersion())) {
                return;
            }
        
            try {
                new DataUpdater().update(arrayDesign);
            } finally {
                try {
                    proxies.remove(arrayDesign);
                    proxy.close();
                } catch (Throwable t) {
                }
            }
        }
    }

    public NetCDFStatisticsCreator getStatisticsCreator(ArrayDesign arrayDesign) {
        return new NetCDFStatisticsCreator(atlasDataDAO, experiment, arrayDesign);
    }

    /*
     * This method returns samples in the order they are stored in netcdf file.
     * While this order is important we have to use this method,
     * in future it would be replaced by Experiment method.
     */
    public List<Sample> getSamples(ArrayDesign arrayDesign) throws AtlasDataException {
        final String[] sampleAccessions = getProxy(arrayDesign).getSampleAccessions();
        final ArrayList<Sample> samples = new ArrayList<Sample>(sampleAccessions.length);
        for (String accession : sampleAccessions) {
            samples.add(experiment.getSample(accession));
        }
        return samples;
    }

    /*
     * This method returns assays in the order they are stored in netcdf file.
     * While this order is important we have to use this method,
     * in future it would be replaced by Experiment method.
     */
    public List<Assay> getAssays(ArrayDesign arrayDesign) throws AtlasDataException {
        final String[] assayAccessions = getProxy(arrayDesign).getAssayAccessions();
        final ArrayList<Assay> assays = new ArrayList<Assay>(assayAccessions.length);
        for (String accession : assayAccessions) {
            assays.add(experiment.getAssay(accession));
        }
        return assays;
    }

    public int[][] getSamplesToAssays(ArrayDesign arrayDesign) throws AtlasDataException {
        return getProxy(arrayDesign).getSamplesToAssays();
    }<commit_msg>an exception is case assays list in netcdf and in database are inconsistent<commit_after>/*
 * Copyright 2008-2010 Microarray Informatics Team, EMBL-European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *
 * For further details of the Gene Expression Atlas project, including source code,
 * downloads and documentation, please see:
 *
 * http://gxa.github.com/gxa
 */

package uk.ac.ebi.gxa.data;

import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.primitives.Floats;
import uk.ac.ebi.gxa.utils.CollectionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.ac.ebi.microarray.atlas.model.*;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.*;

public class ExperimentWithData {
    private final Logger log = LoggerFactory.getLogger(getClass());

    private final AtlasDataDAO atlasDataDAO;
    private final Experiment experiment;

    private final Map<ArrayDesign,DataProxy> proxies = new HashMap<ArrayDesign,DataProxy>();

    // cached data
    private final Map<ArrayDesign, String[]> designElementAccessions = new HashMap<ArrayDesign, String[]>();

    ExperimentWithData(@Nonnull AtlasDataDAO atlasDataDAO, @Nonnull Experiment experiment) {
        this.atlasDataDAO = atlasDataDAO;
        this.experiment = experiment;
    }

    public Experiment getExperiment() {
        return experiment;
    }

    /**
     * @param criteria the criteria to choose arrayDesign
     * @return first arrayDesign used in experiment, that matches criteria;
     *         or null if no arrayDesign has been found
     */
    public ArrayDesign findArrayDesign(Predicate<ArrayDesign> criteria) {
        for (ArrayDesign ad : experiment.getArrayDesigns()) {
            if (criteria.apply(ad)) {
                return ad;
            }
        }
        return null;
    }

    AtlasDataDAO getDataDAO() {
        return atlasDataDAO;
    }

    DataProxy getProxy(ArrayDesign arrayDesign) throws AtlasDataException {
        DataProxy p = proxies.get(arrayDesign);
        if (p == null) {
            p = atlasDataDAO.createDataProxy(experiment, arrayDesign);
            proxies.put(arrayDesign, p);
        }
        return p;
    }

    public NetCDFDataCreator getDataCreator(ArrayDesign arrayDesign) {
        return new NetCDFDataCreator(atlasDataDAO, experiment, arrayDesign);
    }

    public void updateAllData() throws AtlasDataException {
        final DataUpdater updater = new DataUpdater();
        for (ArrayDesign arrayDesign : experiment.getArrayDesigns()) {
            updater.update(arrayDesign);
        }
    }

    public void updateDataToNewestVersion() throws AtlasDataException {
        for (ArrayDesign arrayDesign : experiment.getArrayDesigns()) {
            final DataProxy proxy = getProxy(arrayDesign);
            if ("2.0".equals(proxy.getVersion())) {
                return;
            }
        
            try {
                new DataUpdater().update(arrayDesign);
            } finally {
                try {
                    proxies.remove(arrayDesign);
                    proxy.close();
                } catch (Throwable t) {
                }
            }
        }
    }

    public NetCDFStatisticsCreator getStatisticsCreator(ArrayDesign arrayDesign) {
        return new NetCDFStatisticsCreator(atlasDataDAO, experiment, arrayDesign);
    }

    /*
     * This method returns samples in the order they are stored in netcdf file.
     * While this order is important we have to use this method,
     * in future it would be replaced by Experiment method.
     */
    public List<Sample> getSamples(ArrayDesign arrayDesign) throws AtlasDataException {
        final String[] sampleAccessions = getProxy(arrayDesign).getSampleAccessions();
        final ArrayList<Sample> samples = new ArrayList<Sample>(sampleAccessions.length);
        for (String accession : sampleAccessions) {
            samples.add(experiment.getSample(accession));
        }
        return samples;
    }

    /*
     * This method returns assays in the order they are stored in netcdf file.
     * While this order is important we have to use this method,
     * in future it would be replaced by Experiment method.
     */
    public List<Assay> getAssays(ArrayDesign arrayDesign) throws AtlasDataException {
        final String[] assayAccessions = getProxy(arrayDesign).getAssayAccessions();

        final Map<String,Assay> experimentAssays = new HashMap<String,Assay>();
        for (Assay a : experiment.getAssaysForDesign(arrayDesign)) {
            experimentAssays.put(a.getAccession(), a);
        }
        if (assayAccessions.length != experimentAssays.size()) {
            throw new AtlasDataException("Experiment " + experiment.getAccession() + "/" + arrayDesign.getAccession() + " contains " + experimentAssays.size() + " assays but data storage contains " + assayAccessions.length);
        }

        final ArrayList<Assay> assays = new ArrayList<Assay>(assayAccessions.length);
        for (String accession : assayAccessions) {
            final Assay a = experimentAssays.get(accession);
            if (a == null) {
                throw new AtlasDataException("Experiment " + experiment.getAccession() + "/" + arrayDesign.getAccession() + " does not contain an assay with accession " + accession + " that is mentioned in data storage");
            }
            assays.add(a);
        }
        return assays;
    }

    public int[][] getSamplesToAssays(ArrayDesign arrayDesign) throws AtlasDataException {
        return getProxy(arrayDesign).getSamplesToAssays();
    }
{'fraction_non_alphanumeric': 0.08405191185436808, 'fraction_numerical': 0.002787213657346921, 'mean_word_length': 2.8324432576769025, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3142815', 'n_tokens_mistral': 3102, 'n_tokens_neox': 2963, 'n_words': 887}