<commit_before>/*
 * Copyright 2008-2010 Microarray Informatics Team, EMBL-European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *
 * For further details of the Gene Expression Atlas project, including source code,
 * downloads and documentation, please see:
 *
 * http://gxa.github.com/gxa
 */

package uk.ac.ebi.gxa.loader.steps;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.MAGETABInvestigation;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.*;
import uk.ac.ebi.arrayexpress2.magetab.utils.SDRFUtils;
import uk.ac.ebi.gxa.analytics.compute.AtlasComputeService;
import uk.ac.ebi.gxa.analytics.compute.ComputeTask;
import uk.ac.ebi.gxa.analytics.compute.RUtil;
import uk.ac.ebi.gxa.loader.AtlasLoaderException;
import uk.ac.ebi.gxa.loader.cache.AtlasLoadCache;
import uk.ac.ebi.gxa.loader.datamatrix.DataMatrixFileBuffer;
import uk.ac.ebi.gxa.loader.service.AtlasLoaderServiceListener;
import uk.ac.ebi.gxa.utils.FileUtil;
import uk.ac.ebi.microarray.atlas.model.Assay;
import uk.ac.ebi.rcloud.server.RServices;

import javax.annotation.Nonnull;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.rmi.RemoteException;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import static com.google.common.io.ByteStreams.copy;
import static com.google.common.io.Closeables.closeQuietly;
import static uk.ac.ebi.gxa.utils.FileUtil.deleteDirectory;

/**
 * Experiment loading step that prepares data matrix to be stored into a NetCDF file.
 * Based on the original handlers code by <NAME>.
 *
 * @author <NAME>
 */


public class ArrayDataStep {
    private final static Logger log = LoggerFactory.getLogger(ArrayDataStep.class);

    static final Object SUCCESS_KEY = new Object();

    public static String displayName() {
        return "Processing data matrix";
    }

    private static class RawData {
        final File dataDir;
        final HashMap<String, String> celFiles = new HashMap<String, String>();
        final HashMap<String, Assay> assays = new HashMap<String, Assay>();

        RawData() {
            dataDir = FileUtil.createTempDirectory("atlas-loader");
        }
    }

    private static final Object COPY_FILE_LOCK = new Object();

    private static void copyFile(URL from, File to) throws IOException {
        synchronized (COPY_FILE_LOCK) {
            URLConnection connection = null;
            InputStream is = null;
            OutputStream os = null;
            try {
                connection = from.openConnection();
                is = connection.getInputStream();
                os = new FileOutputStream(to);
                copy(is, os);
            } finally {
                closeQuietly(os);
                closeQuietly(is);
                if (connection != null && connection instanceof HttpURLConnection) {
                    ((HttpURLConnection) connection).disconnect();
                }
            }
        }
    }

    private static final Object EXTRACT_ZIP_LOCK = new Object();

    private static void extractZip(File zipFile, File dir) throws IOException {
        synchronized (EXTRACT_ZIP_LOCK) {
            ZipInputStream zipInputStream = null;
            try {
                zipInputStream = new ZipInputStream(new FileInputStream(zipFile));
                ZipEntry zipEntry = zipInputStream.getNextEntry();
                while (zipEntry != null) {
                    final String entryName = zipEntry.getName();
                    FileOutputStream fileOutputStream = null;

                    try {
                        fileOutputStream = new FileOutputStream(new File(dir, entryName));
                        copy(zipInputStream, fileOutputStream);
                    } finally {
                        closeQuietly(fileOutputStream);
                        zipInputStream.closeEntry();
                    }
                    zipEntry = zipInputStream.getNextEntry();
                }
            } finally {
                closeQuietly(zipInputStream);
            }
        }
    }

    public boolean readArrayData(@Nonnull AtlasComputeService computeService, MAGETABInvestigation investigation, AtlasLoaderServiceListener listener, AtlasLoadCache cache) throws AtlasLoaderException {
        final URL sdrfURL = investigation.SDRF.getLocation();
        final File sdrfDir = new File(sdrfURL.getFile()).getParentFile();
        final HashMap<String, RawData> dataByArrayDesign = new HashMap<String, RawData>();
        final HashMap<String, File> zipFiles = new HashMap<String, File>();

        try {
            boolean useLocalCopy = false;
            listener.setProgress("Loading CEL files");
            for (ArrayDataNode node : investigation.SDRF.lookupNodes(ArrayDataNode.class)) {
                log.info("Found array data matrix node '" + node.getNodeName() + "'");

                final Collection<HybridizationNode> hybridizationNodes = SDRFUtils.findUpstreamNodes(node, HybridizationNode.class);
                final Collection<AssayNode> assayNodes = SDRFUtils.findUpstreamNodes(node, AssayNode.class);
                if (hybridizationNodes.size() + assayNodes.size() != 1) {
                    throw new AtlasLoaderException("ArrayDataNode " + node.getNodeName() + " corresponds to " + (hybridizationNodes.size() + assayNodes.size()) + " assays");
                }
                final HybridizationNode assayNode =
                        hybridizationNodes.size() == 0 ? assayNodes.iterator().next() : hybridizationNodes.iterator().next();
                Assay assay = cache.fetchAssay(assayNode.getNodeName());
                if (assay == null) {<commit_msg>enabled loading data from CEL files located in the same directory as IDF/SDRF files (http://bar.ebi.ac.uk:8080/trac/ticket/2993)<commit_after>/*
 * Copyright 2008-2010 Microarray Informatics Team, EMBL-European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *
 * For further details of the Gene Expression Atlas project, including source code,
 * downloads and documentation, please see:
 *
 * http://gxa.github.com/gxa
 */

package uk.ac.ebi.gxa.loader.steps;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.MAGETABInvestigation;
import uk.ac.ebi.arrayexpress2.magetab.datamodel.sdrf.node.*;
import uk.ac.ebi.arrayexpress2.magetab.utils.SDRFUtils;
import uk.ac.ebi.gxa.analytics.compute.AtlasComputeService;
import uk.ac.ebi.gxa.analytics.compute.ComputeTask;
import uk.ac.ebi.gxa.analytics.compute.RUtil;
import uk.ac.ebi.gxa.loader.AtlasLoaderException;
import uk.ac.ebi.gxa.loader.cache.AtlasLoadCache;
import uk.ac.ebi.gxa.loader.datamatrix.DataMatrixFileBuffer;
import uk.ac.ebi.gxa.loader.service.AtlasLoaderServiceListener;
import uk.ac.ebi.gxa.utils.FileUtil;
import uk.ac.ebi.microarray.atlas.model.Assay;
import uk.ac.ebi.rcloud.server.RServices;

import javax.annotation.Nonnull;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.rmi.RemoteException;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import static com.google.common.io.ByteStreams.copy;
import static com.google.common.io.Closeables.closeQuietly;
import static uk.ac.ebi.gxa.utils.FileUtil.deleteDirectory;

/**
 * Experiment loading step that prepares data matrix to be stored into a NetCDF file.
 * Based on the original handlers code by <NAME>.
 *
 * @author <NAME>
 */


public class ArrayDataStep {
    private final static Logger log = LoggerFactory.getLogger(ArrayDataStep.class);

    static final Object SUCCESS_KEY = new Object();

    public static String displayName() {
        return "Processing data matrix";
    }

    private static class RawData {
        final File dataDir;
        final HashMap<String, String> celFiles = new HashMap<String, String>();
        final HashMap<String, Assay> assays = new HashMap<String, Assay>();

        RawData() {
            dataDir = FileUtil.createTempDirectory("atlas-loader");
        }
    }

    private static final Object COPY_FILE_LOCK = new Object();

    private static void copyFile(URL from, File to) throws IOException {
        synchronized (COPY_FILE_LOCK) {
            URLConnection connection = null;
            InputStream is = null;
            OutputStream os = null;
            try {
                connection = from.openConnection();
                is = connection.getInputStream();
                os = new FileOutputStream(to);
                copy(is, os);
            } finally {
                closeQuietly(os);
                closeQuietly(is);
                if (connection != null && connection instanceof HttpURLConnection) {
                    ((HttpURLConnection) connection).disconnect();
                }
            }
        }
    }

    private static final Object EXTRACT_ZIP_LOCK = new Object();

    private static void extractZip(File zipFile, File dir) throws IOException {
        synchronized (EXTRACT_ZIP_LOCK) {
            ZipInputStream zipInputStream = null;
            try {
                zipInputStream = new ZipInputStream(new FileInputStream(zipFile));
                ZipEntry zipEntry = zipInputStream.getNextEntry();
                while (zipEntry != null) {
                    final String entryName = zipEntry.getName();
                    FileOutputStream fileOutputStream = null;

                    try {
                        fileOutputStream = new FileOutputStream(new File(dir, entryName));
                        copy(zipInputStream, fileOutputStream);
                    } finally {
                        closeQuietly(fileOutputStream);
                        zipInputStream.closeEntry();
                    }
                    zipEntry = zipInputStream.getNextEntry();
                }
            } finally {
                closeQuietly(zipInputStream);
            }
        }
    }

    public boolean readArrayData(@Nonnull AtlasComputeService computeService, MAGETABInvestigation investigation, AtlasLoaderServiceListener listener, AtlasLoadCache cache) throws AtlasLoaderException {
        final URL sdrfURL = investigation.SDRF.getLocation();
        final File sdrfDir = new File(sdrfURL.getFile()).getParentFile();
        final HashMap<String, RawData> dataByArrayDesign = new HashMap<String, RawData>();
        final HashMap<String, File> zipFiles = new HashMap<String, File>();

        try {
            boolean useLocalCopy = true;
            listener.setProgress("Loading CEL files");
            for (ArrayDataNode node : investigation.SDRF.lookupNodes(ArrayDataNode.class)) {
                log.info("Found array data matrix node '" + node.getNodeName() + "'");

                final Collection<HybridizationNode> hybridizationNodes = SDRFUtils.findUpstreamNodes(node, HybridizationNode.class);
                final Collection<AssayNode> assayNodes = SDRFUtils.findUpstreamNodes(node, AssayNode.class);
                if (hybridizationNodes.size() + assayNodes.size() != 1) {
                    throw new AtlasLoaderException("ArrayDataNode " + node.getNodeName() + " corresponds to " + (hybridizationNodes.size() + assayNodes.size()) + " assays");
                }
                final HybridizationNode assayNode =
                        hybridizationNodes.size() == 0 ? assayNodes.iterator().next() : hybridizationNodes.iterator().next();
                Assay assay = cache.fetchAssay(assayNode.getNodeName());
                if (assay == null) {
{'fraction_non_alphanumeric': 0.0932614133833646, 'fraction_numerical': 0.003595997498436523, 'mean_word_length': 2.676149425287356, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4825439', 'n_tokens_mistral': 3520, 'n_tokens_neox': 3313, 'n_words': 870}