<commit_before>/*
 * Copyright 2008-2012 Microarray Informatics Team, EMBL-European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *
 * For further details of the Gene Expression Atlas project, including source code,
 * downloads and documentation, please see:
 *
 * http://gxa.github.com/gxa
 */

package uk.ac.ebi.atlas.model.caches;

import com.google.common.cache.CacheLoader;
import org.springframework.beans.factory.annotation.Value;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.IDF;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.MAGETABInvestigation;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.graph.utils.GraphUtils;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.AssayNode;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.ScanNode;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.SourceNode;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.attribute.CharacteristicsAttribute;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.attribute.FactorValueAttribute;
import uk.ac.ebi.arrayexpress2.magetab.exception.ParseException;
import uk.ac.ebi.arrayexpress2.magetab.parser.MAGETABParser;
import uk.ac.ebi.atlas.model.Experiment;
import uk.ac.ebi.atlas.model.ExperimentRun;
import uk.ac.ebi.atlas.model.readers.AnalysisMethodsTsvReader;
import uk.ac.ebi.atlas.utils.ArrayExpressClient;

import javax.inject.Inject;
import javax.inject.Named;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.MessageFormat;
import java.util.*;

import static com.google.common.base.Preconditions.checkNotNull;

@Named("experimentMetadataLoader")
public class ExperimentMetadataLoader extends CacheLoader<String, Experiment> {

    private static final String ENA_RUN = "ENA_RUN";

    private String idfUrlTemplate;
    private String idfPathTemplate;

    private AnalysisMethodsTsvReader analysisMethodsTsvReader;

    private ArrayExpressClient arrayExpressClient;

    @Inject
    public ExperimentMetadataLoader(
            @Value("#{configuration['experiment.magetab.idf.url.template']}") String idfUrlTemplate
            ,@Value("#{configuration['experiment.magetab.idf.path.template']}") String idfPathTemplate
            , AnalysisMethodsTsvReader analysisMethodsTsvReader, ArrayExpressClient arrayExpressClient) {

        this.idfUrlTemplate = idfUrlTemplate;
        this.idfPathTemplate = idfPathTemplate;
        this.analysisMethodsTsvReader = analysisMethodsTsvReader;
        this.arrayExpressClient = arrayExpressClient;
    }

    @Override
    public Experiment load(String experimentAccession) throws ParseException, IOException {

        MAGETABInvestigation investigation = parseInvestigation(experimentAccession);

        Collection<ScanNode> scanNodes = investigation.SDRF.getNodes(ScanNode.class);

        // TODO: takes first experimental factor type as default
        String experimentalFactorType = investigation.IDF.experimentalFactorType.get(0).replaceAll(" ", "_").toUpperCase();

        Experiment experiment = new Experiment(experimentAccession, arrayExpressClient.fetchExperimentName(experimentAccession)
                , getExperimentRunAccessions(experimentAccession), experimentalFactorType);

        ScanNode firstNode = scanNodes.iterator().next();

        experiment.setSpecie(extractSpecie(firstNode));

        experiment.addAll(extractExperimentRuns(scanNodes, investigation.IDF));

        return experiment;

    }

    private Set<String> getExperimentRunAccessions(String experimentAccession) throws IOException {
        Map<String, String> analysisMethodsRows = analysisMethodsTsvReader.readAsMap(experimentAccession);

        String[] processedLibraries = analysisMethodsRows.get("Processed libraries").split(",");

        Set<String> experimentRunAccessions = new HashSet<>();

        for (String processedLibrary : processedLibraries) {
            experimentRunAccessions.add(processedLibrary.trim());
        }

        return experimentRunAccessions;
    }

    private String extractSpecie(ScanNode firstScanNode) {
        SourceNode firstSourceNode = GraphUtils.findUpstreamNodes(firstScanNode, SourceNode.class).iterator().next();
        for (CharacteristicsAttribute characteristic : firstSourceNode.characteristics) {
            if (characteristic.type.equalsIgnoreCase("ORGANISM")) {
                return characteristic.getAttributeValue();
            }
        }
        return null;
    }

    public Collection<ExperimentRun> extractExperimentRuns(Collection<ScanNode> scanNodes, IDF idf) throws ParseException {

        Collection<ExperimentRun> experimentRuns = new ArrayList<>();

        for (ScanNode scanNode : scanNodes) {

            if (scanNode.comments.keySet().contains(ENA_RUN)) {
                ExperimentRun run = buildExperimentRun(scanNode, idf);
                experimentRuns.add(run);
            }
        }
        return experimentRuns;
    }

    //Required for testability - will be overridden to inject mock
    MAGETABInvestigation parseInvestigation(String experimentAccession) throws ParseException, IOException {

        String idfFileLocation = MessageFormat.format(idfPathTemplate, experimentAccession);
        MAGETABParser<MAGETABInvestigation> mageTabParser = new MAGETABParser<>();
        File idfFile = new File(idfFileLocation);
        if (idfFile.exists()){
            return mageTabParser.parse(idfFile);
        } else {
            URL idfFileURL = new URL(MessageFormat.format(idfUrlTemplate, experimentAccession));
            return mageTabParser.parse(idfFileURL);
        }

    }

    ExperimentRun buildExperimentRun(ScanNode scanNode, IDF idf) {

        ExperimentRun run = new ExperimentRun(scanNode.comments.get(ENA_RUN));

        Collection<AssayNode> assayNodes = GraphUtils.findUpstreamNodes(scanNode, AssayNode.class);

        if (assayNodes.size() != 1) {
            throw new IllegalStateException("No assay corresponds to ENA run " + run.getRunAccession());
        }

        AssayNode assayNode = assayNodes.iterator().next();

        for (FactorValueAttribute factorValueAttribute : assayNode.factorValues) {

            String factorType = null;

            String factorName = factorValueAttribute.type;

            List<String> experimentalFactorNames = idf.experimentalFactorName;
            for (int i = 0; i < experimentalFactorNames.size(); i++) {
                if (experimentalFactorNames.get(i).equals(factorValueAttribute.type)) {
                    if (idf.experimentalFactorType.size() > i) {
                        factorType = idf.experimentalFactorType.get(i);
                    }
                }
            }
            if (factorType == null) {
                factorType = factorName;
            }

            run.addFactorValue(factorType, factorName, factorValueAttribute.getAttributeValue().toLowerCase());


        }

        return run;
    }


}
<commit_msg>Now the application is fully testable also without any network connection - in case of ArrayExpress connection error experiment name is set to a default error string<commit_after>/*
 * Copyright 2008-2012 Microarray Informatics Team, EMBL-European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *
 * For further details of the Gene Expression Atlas project, including source code,
 * downloads and documentation, please see:
 *
 * http://gxa.github.com/gxa
 */

package uk.ac.ebi.atlas.model.caches;

import com.google.common.cache.CacheLoader;
import org.apache.log4j.Logger;
import org.apache.velocity.tools.view.servlet.ServletLogger;
import org.springframework.beans.factory.annotation.Value;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.IDF;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.MAGETABInvestigation;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.graph.utils.GraphUtils;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.AssayNode;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.ScanNode;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.SourceNode;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.attribute.CharacteristicsAttribute;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.attribute.FactorValueAttribute;
import uk.ac.ebi.arrayexpress2.magetab.exception.ParseException;
import uk.ac.ebi.arrayexpress2.magetab.parser.MAGETABParser;
import uk.ac.ebi.atlas.model.Experiment;
import uk.ac.ebi.atlas.model.ExperimentRun;
import uk.ac.ebi.atlas.model.readers.AnalysisMethodsTsvReader;
import uk.ac.ebi.atlas.utils.ArrayExpressClient;

import javax.inject.Inject;
import javax.inject.Named;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.MessageFormat;
import java.util.*;

import static com.google.common.base.Preconditions.checkNotNull;

@Named("experimentMetadataLoader")
public class ExperimentMetadataLoader extends CacheLoader<String, Experiment> {

    private static final Logger LOGGER = Logger.getLogger(ExperimentMetadataLoader.class);

    private static final String ENA_RUN = "ENA_RUN";

    private String idfUrlTemplate;
    private String idfPathTemplate;

    private AnalysisMethodsTsvReader analysisMethodsTsvReader;

    private ArrayExpressClient arrayExpressClient;

    @Inject
    public ExperimentMetadataLoader(
            @Value("#{configuration['experiment.magetab.idf.url.template']}") String idfUrlTemplate
            ,@Value("#{configuration['experiment.magetab.idf.path.template']}") String idfPathTemplate
            , AnalysisMethodsTsvReader analysisMethodsTsvReader, ArrayExpressClient arrayExpressClient) {

        this.idfUrlTemplate = idfUrlTemplate;
        this.idfPathTemplate = idfPathTemplate;
        this.analysisMethodsTsvReader = analysisMethodsTsvReader;
        this.arrayExpressClient = arrayExpressClient;
    }

    @Override
    public Experiment load(String experimentAccession) throws ParseException, IOException {

        MAGETABInvestigation investigation = parseInvestigation(experimentAccession);

        Collection<ScanNode> scanNodes = investigation.SDRF.getNodes(ScanNode.class);

        // TODO: takes first experimental factor type as default
        String experimentalFactorType = investigation.IDF.experimentalFactorType.get(0).replaceAll(" ", "_").toUpperCase();

        String experimentName = fetchExperimentName(experimentAccession);

        Experiment experiment = new Experiment(experimentAccession, experimentName
                , getExperimentRunAccessions(experimentAccession), experimentalFactorType);

        ScanNode firstNode = scanNodes.iterator().next();

        experiment.setSpecie(extractSpecie(firstNode));

        experiment.addAll(extractExperimentRuns(scanNodes, investigation.IDF));

        return experiment;

    }

    private String fetchExperimentName(String experimentAccession) {
        try{
            return arrayExpressClient.fetchExperimentName(experimentAccession);
        } catch(Exception e) {
            LOGGER.error(e.getMessage(), e);
            return "Error connecting to ArrayExpress!";
        }
    }

    private Set<String> getExperimentRunAccessions(String experimentAccession) throws IOException {
        Map<String, String> analysisMethodsRows = analysisMethodsTsvReader.readAsMap(experimentAccession);

        String[] processedLibraries = analysisMethodsRows.get("Processed libraries").split(",");

        Set<String> experimentRunAccessions = new HashSet<>();

        for (String processedLibrary : processedLibraries) {
            experimentRunAccessions.add(processedLibrary.trim());
        }

        return experimentRunAccessions;
    }

    private String extractSpecie(ScanNode firstScanNode) {
        SourceNode firstSourceNode = GraphUtils.findUpstreamNodes(firstScanNode, SourceNode.class).iterator().next();
        for (CharacteristicsAttribute characteristic : firstSourceNode.characteristics) {
            if (characteristic.type.equalsIgnoreCase("ORGANISM")) {
                return characteristic.getAttributeValue();
            }
        }
        return null;
    }

    public Collection<ExperimentRun> extractExperimentRuns(Collection<ScanNode> scanNodes, IDF idf) throws ParseException {

        Collection<ExperimentRun> experimentRuns = new ArrayList<>();

        for (ScanNode scanNode : scanNodes) {

            if (scanNode.comments.keySet().contains(ENA_RUN)) {
                ExperimentRun run = buildExperimentRun(scanNode, idf);
                experimentRuns.add(run);
            }
        }
        return experimentRuns;
    }

    //Required for testability - will be overridden to inject mock
    MAGETABInvestigation parseInvestigation(String experimentAccession) throws ParseException, IOException {

        String idfFileLocation = MessageFormat.format(idfPathTemplate, experimentAccession);
        MAGETABParser<MAGETABInvestigation> mageTabParser = new MAGETABParser<>();
        File idfFile = new File(idfFileLocation);
        if (idfFile.exists()){
            return mageTabParser.parse(idfFile);
        } else {
            URL idfFileURL = new URL(MessageFormat.format(idfUrlTemplate, experimentAccession));
            return mageTabParser.parse(idfFileURL);
        }

    }

    ExperimentRun buildExperimentRun(ScanNode scanNode, IDF idf) {

        ExperimentRun run = new ExperimentRun(scanNode.comments.get(ENA_RUN));

        Collection<AssayNode> assayNodes = GraphUtils.findUpstreamNodes(scanNode, AssayNode.class);

        if (assayNodes.size() != 1) {
            throw new IllegalStateException("No assay corresponds to ENA run " + run.getRunAccession());
        }

        AssayNode assayNode = assayNodes.iterator().next();

        for (FactorValueAttribute factorValueAttribute : assayNode.factorValues) {

            String factorType = null;

            String factorName = factorValueAttribute.type;

            List<String> experimentalFactorNames = idf.experimentalFactorName;
            for (int i = 0; i < experimentalFactorNames.size(); i++) {
                if (experimentalFactorNames.get(i).equals(factorValueAttribute.type)) {
                    if (idf.experimentalFactorType.size() > i) {
                        factorType = idf.experimentalFactorType.get(i);
                    }
                }
            }
            if (factorType == null) {
                factorType = factorName;
            }

            run.addFactorValue(factorType, factorName, factorValueAttribute.getAttributeValue().toLowerCase());


        }

        return run;
    }


}

{'fraction_non_alphanumeric': 0.0888315285822211, 'fraction_numerical': 0.003210778141526064, 'mean_word_length': 3.554185779816514, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7323596', 'n_tokens_mistral': 4410, 'n_tokens_neox': 4200, 'n_words': 900}