<commit_before>/*
   Copyright (C) 2005-2012, by the President and Fellows of Harvard College.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   Dataverse Network - A web application to share, preserve and analyze research data.
   Developed at the Institute for Quantitative Social Science, Harvard University.
   Version 3.0.
*/

package edu.harvard.iq.dataverse.ingest;

import edu.harvard.iq.dataverse.ControlledVocabularyValue;
import edu.harvard.iq.dataverse.datavariable.VariableServiceBean;
import edu.harvard.iq.dataverse.DatasetServiceBean;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFileServiceBean;
import edu.harvard.iq.dataverse.DataTable;
import edu.harvard.iq.dataverse.DatasetField;
import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
import edu.harvard.iq.dataverse.DatasetFieldType;
import edu.harvard.iq.dataverse.DatasetFieldValue;
import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
import edu.harvard.iq.dataverse.DatasetVersion;
import edu.harvard.iq.dataverse.DataverseUser;
import edu.harvard.iq.dataverse.FileMetadata;
import edu.harvard.iq.dataverse.MetadataBlock;
import edu.harvard.iq.dataverse.dataaccess.DataAccessObject;
import edu.harvard.iq.dataverse.dataaccess.DataAccessOption;
import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
import edu.harvard.iq.dataverse.dataaccess.TabularSubsetGenerator;
import edu.harvard.iq.dataverse.datavariable.SummaryStatistic;
import edu.harvard.iq.dataverse.datavariable.DataVariable;
import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataExtractor;
import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataIngest;
import edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.fits.FITSFileMetadataExtractor;
import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta.DTAFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta.DTAFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.rdata.RDATAFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.rdata.RDATAFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.csv.CSVFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.csv.CSVFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.xlsx.XLSXFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.xlsx.XLSXFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReaderSpi;
import edu.harvard.iq.dataverse.util.FileUtil;
import edu.harvard.iq.dataverse.util.MD5Checksum;
import edu.harvard.iq.dataverse.util.ShapefileHandler;
import edu.harvard.iq.dataverse.util.SumStatCalculator;
import edu.harvard.iq.dvn.unf.*;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.Stateless;
import javax.inject.Named;
import javax.jms.Queue;
import javax.jms.QueueConnectionFactory;
import javax.annotation.Resource;
import javax.jms.JMSException;
import javax.jms.QueueConnection;
import javax.jms.QueueSender;
import javax.jms.QueueSession;
import javax.jms.Message;
import javax.faces.bean.ManagedBean;
import org.primefaces.push.PushContext;
import org.primefaces.push.PushContextFactory;
import javax.faces.application.FacesMessage;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.io.FileUtils;

/**
 *
 * @author <NAME>
 * dataverse 4.0
 * New service for handling ingest tasks
 * 
 */
@ManagedBean
@Stateless
@Named
public class IngestServiceBean {
    private static final Logger logger = Logger.getLogger(IngestServiceBean.class.getCanonicalName());
    @EJB
    VariableServiceBean variableService;
    @EJB 
    DatasetServiceBean datasetService;
    @EJB
    DatasetFieldServiceBean fieldService;
    @EJB
    DataFileServiceBean fileService; 

    @Resource(mappedName = "jms/DataverseIngest")
    Queue queue;
    @Resource(mappedName = "jms/IngestQueueConnectionFactory")
    QueueConnectionFactory factory;
    
    private static final String MIME_TYPE_STATA = "application/x-stata";
    private static final String MIME_TYPE_RDATA = "application/x-rlang-transport";
    private static final String MIME_TYPE_CSV   = "text/csv";
    private static final String MIME_TYPE_CSV_ALT = "text/comma-separated-values";
    private static final String MIME_TYPE_XLSX  = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
    private static final String MIME_TYPE_SPSS_SAV = "application/x-spss-sav";
    private static final String MIME_TYPE_SPSS_POR = "application/x-spss-por";
    
    private static final String MIME_TYPE_TAB   = "text/tab-separated-values";
    
    private static final String MIME_TYPE_FITS  = "application/fits";
    
    private static final String MIME_TYPE_UNDETERMINED_DEFAULT = "application/octet-stream";
    private static String dateTimeFormat_ymdhmsS = "yyyy-MM-dd HH:mm:ss.SSS";
    private static String dateFormat_ymd = "yyyy-MM-dd";
      
    @Deprecated
    // All the parts of the app should use the createDataFiles() method instead, 
    // that returns a list of DataFiles. 
    public DataFile createDataFile(DatasetVersion version, InputStream inputStream, String fileName, String contentType) throws IOException {
        List<DataFile> fileList = createDataFiles(version, inputStream, fileName, contentType);
        
        if (fileList == null) {
            return null; 
        }
        
        return fileList.get(0);
    }
    
    public List<DataFile> createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String contentType) throws IOException {
        List<DataFile> datafiles = new ArrayList<DataFile>(); 
        
        // save the file, in the temporary location for now: 
        Path tempFile = null; 
        
        
        if (getFilesTempDirectory() != null) {
            tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload");
            // "temporary" location is the key here; this is why we are not using 
            // the DataStore framework for this - the assumption is that 
            // temp files will always be stored on the local filesystem. 
            //          -- L.A. Jul. 2014
            logger.fine("Will attempt to save the file as: " + tempFile.toString());
            Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING);
        } else {
            throw new IOException ("Temp directory is not configured.");
        }
        
        // Let's try our own utilities (Jhove, etc.) to determine the file type 
        // of the uploaded file. (We may already have a mime type supplied for this
        // file - maybe the type that the browser recognized on upload; or, if 
        // it's a harvest, maybe the remote server has already given us the type
        // for this file... with our own type utility we may or may not do better 
        // than the type supplied:
        //  -- L.A. 
        String recognizedType = null;
        String finalType = null; 
        try {
            recognizedType = FileUtil.determineFileType(tempFile.toFile(), fileName);
            logger.fine("File utility recognized the file as " + recognizedType);
            if (recognizedType != null && !recognizedType.equals("")) {
                // is it any better than the type that was supplied to us,
                // if any?
                if (contentType == null
                        || contentType.equals("")
                        || contentType.equalsIgnoreCase(MIME_TYPE_UNDETERMINED_DEFAULT)
                        || recognizedType.equals("application/fits-gzipped")
                        || recognizedType.equals(ShapefileHandler.SHAPEFILE_FILE_TYPE)) {
                    finalType = recognizedType;
                }
            }
            
        } catch (IOException ex) {
            logger.warning("Failed to run the file utility mime type check on file " + fileName);
        }
        
        if (finalType == null) {
            finalType = (contentType == null || contentType.equals("")) 
                ? MIME_TYPE_UNDETERMINED_DEFAULT
                : contentType;
        }
                
        // A few special cases: 
        
        // if this is a gzipped FITS file, we'll uncompress it, and ingest it as
        // a regular FITS file:
        
        if (finalType.equals("application/fits-gzipped")) {

            InputStream uncompressedIn = null;
            String finalFileName = fileName;
            // if the file name had the ".gz" extension, remove it, 
            // since we are going to uncompress it:
            if (fileName != null && fileName.matches(".*\\.gz$")) {
                finalFileName = fileName.replaceAll("\\.gz$", "");
            }
            
            DataFile datafile = null; 
            try {                
                uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile()));  
                datafile = createSingleDataFile(version, uncompressedIn, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT);
            } catch (IOException ioex) {
                datafile = null;
            } finally {
                if (uncompressedIn != null) {
                    try {uncompressedIn.close();} catch (IOException e) {}
                }
            }
            
            // If we were able to produce an uncompressed file, we'll use it 
            // to create and return a final DataFile; if not, we're not going
            // to do anything - and then a new DataFile will be created further 
            // down, from the original, uncompressed file.
            if (datafile != null) {
                // remove the compressed temp file: 
                try {
                    tempFile.toFile().delete();
                } catch (SecurityException ex) {
                    // (this is very non-fatal)
                    logger.warning("Failed to delete temporary file "+tempFile.toString());
                }
                
                datafiles.add(datafile);
                return datafiles;
            }
                
        // If it's a ZIP file, we are going to unpack it and create multiple 
        // DataFile objects from its contents:
       // } else if (( finalType.equals("application/zip")) || (finalType.equals(ShapefileHandler.SHAPEFILE_FILE_TYPE))) {   
          } else if (finalType.equals("application/zip")) {   
            
            ZipInputStream unZippedIn = null; 
            ZipEntry zipEntry = null; 
            try {
                unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()));

                if (unZippedIn == null) {
                    return null;
                }
                while ((zipEntry = unZippedIn.getNextEntry()) != null) {
                    // Note that some zip entries may be directories - we 
                    // simply skip them:
                    
                    if (!zipEntry.isDirectory()) {

                        String fileEntryName = zipEntry.getName();

                        if (fileEntryName != null && !fileEntryName.equals("")) {

                            fileEntryName = fileEntryName.replaceFirst("^.*[\\/]", "");

                            // Check if it's a "fake" file - a zip archive entry 
                            // created for a MacOS X filesystem element: (these 
                            // start with "._")
                            if (!fileEntryName.startsWith("._")) {
                                // OK, this seems like an OK file entry - we'll try 
                                // to read it and create a DataFile with it:

                                DataFile datafile = createSingleDataFile(version, unZippedIn, fileEntryName, MIME_TYPE_UNDETERMINED_DEFAULT);
                                
                                if (datafile != null) {
                                    // We have created this datafile with the mime type "unknown";
                                    // Now that we have it saved in a temporary location, 
                                    // let's try and determine its real type:
                                    
                                    String tempFileName = getFilesTempDirectory() + "/" + datafile.getFileSystemName();
                                    
                                    try {
                                        recognizedType = FileUtil.determineFileType(new File(tempFileName), fileEntryName);
                                        logger.fine("File utility recognized unzipped file as " + recognizedType);
                                        if (recognizedType != null && !recognizedType.equals("")) {
                                            datafile.setContentType(recognizedType);
                                        }
                                    } catch (IOException ex) {
                                        logger.warning("Failed to run the file utility mime type check on file " + fileName);
                                    }
                                    
                                    datafiles.add(datafile);
                                }
                            }
                        }
                    }
                    unZippedIn.closeEntry(); 
                }
                
            } catch (IOException ioex) {
                // just clear the datafiles list and let 
                // ingest default to creating a single DataFile out
                // of the unzipped file. 
                datafiles.clear();
            } finally {
                if (unZippedIn != null) {
                    try {unZippedIn.close();} catch (Exception zEx) {}
                }
            }
            if (datafiles.size() > 0) {
                return datafiles;
            }
            
        } else if (finalType.equals(ShapefileHandler.SHAPEFILE_FILE_TYPE)) {
            // Shape files may have to be split into multiple files, 
            // one zip archive per each complete set of shape files:
                       
            //File rezipFolder = new File(this.getFilesTempDirectory());
            File rezipFolder = this.getShapefileUnzipTempDirectory();
            
            IngestServiceShapefileHelper shpIngestHelper;
            shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder);

            boolean didProcessWork = shpIngestHelper.processFile();
            if (!(didProcessWork)){            
                logger.severe("Processing of zipped shapefile failed.");
                return null;
            }
            for (File finalFile : shpIngestHelper.getFinalRezippedFiles()){
                FileInputStream finalFileInputStream = new FileInputStream(finalFile);
                finalType = this.getContentType(finalFile);
                if (finalType==null){
                    logger.warning("Content type is null; but should default to 'MIME_TYPE_UNDETERMINED_DEFAULT'");
                    continue; 
                }               
                DataFile new_datafile = createSingleDataFile(version, finalFileInputStream, finalFile.getName(), finalType);
                if (new_datafile != null) {
                  datafiles.add(new_datafile);
                }
                finalFileInputStream.close();                
             
            }
            
            // Delete the temp directory used for unzipping
            logger.fine("Delete temp shapefile unzip directory: " + rezipFolder.getAbsolutePath());
            FileUtils.deleteDirectory(rezipFolder);

            // Delete rezipped files
            for (File finalFile : shpIngestHelper.getFinalRezippedFiles()){
                if (finalFile.isFile()){
                    finalFile.delete();
                }
            }
            
            if (datafiles.size() > 0) {
                return datafiles;
            }
            return null;
           
        }

        
        // Finally, if none of the special cases above were applicable (or 
        // if we were unable to unpack an uploaded file, etc.), we'll just 
        // create and return a single DataFile:
        // (Note that we are passing null for the InputStream; that's because
        // we already have the file saved; we'll just need to rename it, below)
        
        DataFile datafile = createSingleDataFile(version, null, fileName, finalType);;
        
        if (datafile != null) {
            fileService.generateStorageIdentifier(datafile);
            if (!tempFile.toFile().renameTo(new File(getFilesTempDirectory() + "/" + datafile.getFileSystemName()))) {
                return null; 
            }
            
            // MD5:
            MD5Checksum md5Checksum = new MD5Checksum();
            try {
                datafile.setmd5(md5Checksum.CalculateMD5(getFilesTempDirectory() + "/" + datafile.getFileSystemName()));
            } catch (Exception md5ex) {
                logger.warning("Could not calculate MD5 signature for new file " + fileName);
            }
        
            datafiles.add(datafile);
            return datafiles;
        }
        
        return null;
    }   // end createDataFiles
    
    
    /**
     *  Returns a content type string for a FileObject
     * 
     */
    private String getContentType(File fileObject){
        if (fileObject==null){
            return null;
        }
        String contentType;
        try {
            contentType = FileUtil.determineFileType(fileObject, fileObject.getName());
        } catch (IOException ex) {
            logger.info("FileUtil.determineFileType failed for file with name: " + fileObject.getName());
            contentType = null;
        }

        if ((contentType==null)||(contentType=="")){
            contentType = "MIME_TYPE_UNDETERMINED_DEFAULT";
        }
        return contentType;
        
    }
    /* 
     * This method creates a DataFile, and also saves the bytes from the suppplied 
     * InputStream in the temporary location. 
     * This method should only be called by the upper-level methods that handle 
     * file upload and creation for individual use cases - a single file upload, 
     * an upload of a zip archive that needs to be unpacked and turned into 
     * individual files, etc., and once the file name and mime type have already 
     * been figured out. 
    */
    
    private DataFile createSingleDataFile(DatasetVersion version, InputStream inputStream, String fileName, String contentType) {

        DataFile datafile = new DataFile(contentType);
        FileMetadata fmd = new FileMetadata();
        
        fmd.setLabel(fileName);

        datafile.setOwner(version.getDataset());
        fmd.setDataFile(datafile);
        datafile.getFileMetadatas().add(fmd);
        if (version.getFileMetadatas() == null) {
            version.setFileMetadatas(new ArrayList());
        }
        version.getFileMetadatas().add(fmd);
        fmd.setDatasetVersion(version);
        version.getDataset().getFiles().add(datafile);

        // And save the file - but only if the InputStream is not null; 
        // (the temp file may be saved already - if this is a single
        // file upload case - and in that case this method gets called <commit_msg>Added code to automatically resolve duplicate file names, per #357.<commit_after>/*
   Copyright (C) 2005-2012, by the President and Fellows of Harvard College.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   Dataverse Network - A web application to share, preserve and analyze research data.
   Developed at the Institute for Quantitative Social Science, Harvard University.
   Version 3.0.
*/

package edu.harvard.iq.dataverse.ingest;

import edu.harvard.iq.dataverse.ControlledVocabularyValue;
import edu.harvard.iq.dataverse.datavariable.VariableServiceBean;
import edu.harvard.iq.dataverse.DatasetServiceBean;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFileServiceBean;
import edu.harvard.iq.dataverse.DataTable;
import edu.harvard.iq.dataverse.DatasetField;
import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
import edu.harvard.iq.dataverse.DatasetFieldType;
import edu.harvard.iq.dataverse.DatasetFieldValue;
import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
import edu.harvard.iq.dataverse.DatasetVersion;
import edu.harvard.iq.dataverse.DataverseUser;
import edu.harvard.iq.dataverse.FileMetadata;
import edu.harvard.iq.dataverse.MetadataBlock;
import edu.harvard.iq.dataverse.dataaccess.DataAccessObject;
import edu.harvard.iq.dataverse.dataaccess.DataAccessOption;
import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
import edu.harvard.iq.dataverse.dataaccess.TabularSubsetGenerator;
import edu.harvard.iq.dataverse.datavariable.SummaryStatistic;
import edu.harvard.iq.dataverse.datavariable.DataVariable;
import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataExtractor;
import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataIngest;
import edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.fits.FITSFileMetadataExtractor;
import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta.DTAFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta.DTAFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.rdata.RDATAFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.rdata.RDATAFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.csv.CSVFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.csv.CSVFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.xlsx.XLSXFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.xlsx.XLSXFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReaderSpi;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReaderSpi;
import edu.harvard.iq.dataverse.util.FileUtil;
import edu.harvard.iq.dataverse.util.MD5Checksum;
import edu.harvard.iq.dataverse.util.ShapefileHandler;
import edu.harvard.iq.dataverse.util.SumStatCalculator;
import edu.harvard.iq.dvn.unf.*;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.Stateless;
import javax.inject.Named;
import javax.jms.Queue;
import javax.jms.QueueConnectionFactory;
import javax.annotation.Resource;
import javax.jms.JMSException;
import javax.jms.QueueConnection;
import javax.jms.QueueSender;
import javax.jms.QueueSession;
import javax.jms.Message;
import javax.faces.bean.ManagedBean;
import org.primefaces.push.PushContext;
import org.primefaces.push.PushContextFactory;
import javax.faces.application.FacesMessage;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.io.FileUtils;

/**
 *
 * @author <NAME>
 * dataverse 4.0
 * New service for handling ingest tasks
 * 
 */
@ManagedBean
@Stateless
@Named
public class IngestServiceBean {
    private static final Logger logger = Logger.getLogger(IngestServiceBean.class.getCanonicalName());
    @EJB
    VariableServiceBean variableService;
    @EJB 
    DatasetServiceBean datasetService;
    @EJB
    DatasetFieldServiceBean fieldService;
    @EJB
    DataFileServiceBean fileService; 

    @Resource(mappedName = "jms/DataverseIngest")
    Queue queue;
    @Resource(mappedName = "jms/IngestQueueConnectionFactory")
    QueueConnectionFactory factory;
    
    private static final String MIME_TYPE_STATA = "application/x-stata";
    private static final String MIME_TYPE_RDATA = "application/x-rlang-transport";
    private static final String MIME_TYPE_CSV   = "text/csv";
    private static final String MIME_TYPE_CSV_ALT = "text/comma-separated-values";
    private static final String MIME_TYPE_XLSX  = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
    private static final String MIME_TYPE_SPSS_SAV = "application/x-spss-sav";
    private static final String MIME_TYPE_SPSS_POR = "application/x-spss-por";
    
    private static final String MIME_TYPE_TAB   = "text/tab-separated-values";
    
    private static final String MIME_TYPE_FITS  = "application/fits";
    
    private static final String MIME_TYPE_UNDETERMINED_DEFAULT = "application/octet-stream";
    private static String dateTimeFormat_ymdhmsS = "yyyy-MM-dd HH:mm:ss.SSS";
    private static String dateFormat_ymd = "yyyy-MM-dd";
      
    @Deprecated
    // All the parts of the app should use the createDataFiles() method instead, 
    // that returns a list of DataFiles. 
    public DataFile createDataFile(DatasetVersion version, InputStream inputStream, String fileName, String contentType) throws IOException {
        List<DataFile> fileList = createDataFiles(version, inputStream, fileName, contentType);
        
        if (fileList == null) {
            return null; 
        }
        
        return fileList.get(0);
    }
    
    public List<DataFile> createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String contentType) throws IOException {
        List<DataFile> datafiles = new ArrayList<DataFile>(); 
        
        // save the file, in the temporary location for now: 
        Path tempFile = null; 
        
        
        if (getFilesTempDirectory() != null) {
            tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload");
            // "temporary" location is the key here; this is why we are not using 
            // the DataStore framework for this - the assumption is that 
            // temp files will always be stored on the local filesystem. 
            //          -- <NAME>. 2014
            logger.fine("Will attempt to save the file as: " + tempFile.toString());
            Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING);
        } else {
            throw new IOException ("Temp directory is not configured.");
        }
        
        // Let's try our own utilities (Jhove, etc.) to determine the file type 
        // of the uploaded file. (We may already have a mime type supplied for this
        // file - maybe the type that the browser recognized on upload; or, if 
        // it's a harvest, maybe the remote server has already given us the type
        // for this file... with our own type utility we may or may not do better 
        // than the type supplied:
        //  -- L.A. 
        String recognizedType = null;
        String finalType = null; 
        try {
            recognizedType = FileUtil.determineFileType(tempFile.toFile(), fileName);
            logger.fine("File utility recognized the file as " + recognizedType);
            if (recognizedType != null && !recognizedType.equals("")) {
                // is it any better than the type that was supplied to us,
                // if any?
                if (contentType == null
                        || contentType.equals("")
                        || contentType.equalsIgnoreCase(MIME_TYPE_UNDETERMINED_DEFAULT)
                        || recognizedType.equals("application/fits-gzipped")
                        || recognizedType.equals(ShapefileHandler.SHAPEFILE_FILE_TYPE)) {
                    finalType = recognizedType;
                }
            }
            
        } catch (IOException ex) {
            logger.warning("Failed to run the file utility mime type check on file " + fileName);
        }
        
        if (finalType == null) {
            finalType = (contentType == null || contentType.equals("")) 
                ? MIME_TYPE_UNDETERMINED_DEFAULT
                : contentType;
        }
                
        // A few special cases: 
        
        // if this is a gzipped FITS file, we'll uncompress it, and ingest it as
        // a regular FITS file:
        
        if (finalType.equals("application/fits-gzipped")) {

            InputStream uncompressedIn = null;
            String finalFileName = fileName;
            // if the file name had the ".gz" extension, remove it, 
            // since we are going to uncompress it:
            if (fileName != null && fileName.matches(".*\\.gz$")) {
                finalFileName = fileName.replaceAll("\\.gz$", "");
            }
            
            DataFile datafile = null; 
            try {                
                uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile()));  
                datafile = createSingleDataFile(version, uncompressedIn, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT);
            } catch (IOException ioex) {
                datafile = null;
            } finally {
                if (uncompressedIn != null) {
                    try {uncompressedIn.close();} catch (IOException e) {}
                }
            }
            
            // If we were able to produce an uncompressed file, we'll use it 
            // to create and return a final DataFile; if not, we're not going
            // to do anything - and then a new DataFile will be created further 
            // down, from the original, uncompressed file.
            if (datafile != null) {
                // remove the compressed temp file: 
                try {
                    tempFile.toFile().delete();
                } catch (SecurityException ex) {
                    // (this is very non-fatal)
                    logger.warning("Failed to delete temporary file "+tempFile.toString());
                }
                
                datafiles.add(datafile);
                return datafiles;
            }
                
        // If it's a ZIP file, we are going to unpack it and create multiple 
        // DataFile objects from its contents:
       // } else if (( finalType.equals("application/zip")) || (finalType.equals(ShapefileHandler.SHAPEFILE_FILE_TYPE))) {   
          } else if (finalType.equals("application/zip")) {   
            
            ZipInputStream unZippedIn = null; 
            ZipEntry zipEntry = null; 
            try {
                unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()));

                if (unZippedIn == null) {
                    return null;
                }
                while ((zipEntry = unZippedIn.getNextEntry()) != null) {
                    // Note that some zip entries may be directories - we 
                    // simply skip them:
                    
                    if (!zipEntry.isDirectory()) {

                        String fileEntryName = zipEntry.getName();

                        if (fileEntryName != null && !fileEntryName.equals("")) {

                            fileEntryName = fileEntryName.replaceFirst("^.*[\\/]", "");

                            // Check if it's a "fake" file - a zip archive entry 
                            // created for a MacOS X filesystem element: (these 
                            // start with "._")
                            if (!fileEntryName.startsWith("._")) {
                                // OK, this seems like an OK file entry - we'll try 
                                // to read it and create a DataFile with it:

                                DataFile datafile = createSingleDataFile(version, unZippedIn, fileEntryName, MIME_TYPE_UNDETERMINED_DEFAULT);
                                
                                if (datafile != null) {
                                    // We have created this datafile with the mime type "unknown";
                                    // Now that we have it saved in a temporary location, 
                                    // let's try and determine its real type:
                                    
                                    String tempFileName = getFilesTempDirectory() + "/" + datafile.getFileSystemName();
                                    
                                    try {
                                        recognizedType = FileUtil.determineFileType(new File(tempFileName), fileEntryName);
                                        logger.fine("File utility recognized unzipped file as " + recognizedType);
                                        if (recognizedType != null && !recognizedType.equals("")) {
                                            datafile.setContentType(recognizedType);
                                        }
                                    } catch (IOException ex) {
                                        logger.warning("Failed to run the file utility mime type check on file " + fileName);
                                    }
                                    
                                    datafiles.add(datafile);
                                }
                            }
                        }
                    }
                    unZippedIn.closeEntry(); 
                }
                
            } catch (IOException ioex) {
                // just clear the datafiles list and let 
                // ingest default to creating a single DataFile out
                // of the unzipped file. 
                datafiles.clear();
            } finally {
                if (unZippedIn != null) {
                    try {unZippedIn.close();} catch (Exception zEx) {}
                }
            }
            if (datafiles.size() > 0) {
                return datafiles;
            }
            
        } else if (finalType.equals(ShapefileHandler.SHAPEFILE_FILE_TYPE)) {
            // Shape files may have to be split into multiple files, 
            // one zip archive per each complete set of shape files:
                       
            //File rezipFolder = new File(this.getFilesTempDirectory());
            File rezipFolder = this.getShapefileUnzipTempDirectory();
            
            IngestServiceShapefileHelper shpIngestHelper;
            shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder);

            boolean didProcessWork = shpIngestHelper.processFile();
            if (!(didProcessWork)){            
                logger.severe("Processing of zipped shapefile failed.");
                return null;
            }
            for (File finalFile : shpIngestHelper.getFinalRezippedFiles()){
                FileInputStream finalFileInputStream = new FileInputStream(finalFile);
                finalType = this.getContentType(finalFile);
                if (finalType==null){
                    logger.warning("Content type is null; but should default to 'MIME_TYPE_UNDETERMINED_DEFAULT'");
                    continue; 
                }               
                DataFile new_datafile = createSingleDataFile(version, finalFileInputStream, finalFile.getName(), finalType);
                if (new_datafile != null) {
                  datafiles.add(new_datafile);
                }
                finalFileInputStream.close();                
             
            }
            
            // Delete the temp directory used for unzipping
            logger.fine("Delete temp shapefile unzip directory: " + rezipFolder.getAbsolutePath());
            FileUtils.deleteDirectory(rezipFolder);

            // Delete rezipped files
            for (File finalFile : shpIngestHelper.getFinalRezippedFiles()){
                if (finalFile.isFile()){
                    finalFile.delete();
                }
            }
            
            if (datafiles.size() > 0) {
                return datafiles;
            }
            return null;
           
        }

        
        // Finally, if none of the special cases above were applicable (or 
        // if we were unable to unpack an uploaded file, etc.), we'll just 
        // create and return a single DataFile:
        // (Note that we are passing null for the InputStream; that's because
        // we already have the file saved; we'll just need to rename it, below)
        
        DataFile datafile = createSingleDataFile(version, null, fileName, finalType);;
        
        if (datafile != null) {
            fileService.generateStorageIdentifier(datafile);
            if (!tempFile.toFile().renameTo(new File(getFilesTempDirectory() + "/" + datafile.getFileSystemName()))) {
                return null; 
            }
            
            // MD5:
            MD5Checksum md5Checksum = new MD5Checksum();
            try {
                datafile.setmd5(md5Checksum.CalculateMD5(getFilesTempDirectory() + "/" + datafile.getFileSystemName()));
            } catch (Exception md5ex) {
                logger.warning("Could not calculate MD5 signature for new file " + fileName);
            }
        
            datafiles.add(datafile);
            return datafiles;
        }
        
        return null;
    }   // end createDataFiles
    
    // TODO: 
    // add comments explaining what's going on in the 2 methods below. 
    // -- L.A. 4.0 beta
    private String checkForDuplicateFileNames(DatasetVersion version, String fileName) {
        Set<String> fileNamesExisting = new HashSet<String>();

        Iterator<FileMetadata> fmIt = version.getFileMetadatas().iterator();
        while (fmIt.hasNext()) {
            FileMetadata fm = fmIt.next();
            String existingName = fm.getLabel();
            if (existingName != null) {
                fileNamesExisting.add(existingName);
            }
        }

        while (fileNamesExisting.contains(fileName)) {
            fileName = generateNewFileName(fileName);
        }

        return fileName;
    }
    
    private String generateNewFileName(String fileName) {
        String newName = null;
        String baseName = null; 
        String extension = null; 
        
        int extensionIndex = fileName.lastIndexOf(".");
        if (extensionIndex != -1 ) {
            extension = fileName.substring(extensionIndex+1);
            baseName = fileName.substring(0, extensionIndex);
        } else {
            baseName = fileName; 
        }
        
        if (baseName.matches(".*-[0-9][0-9]*$")) {
            int dashIndex = baseName.lastIndexOf("-");
            String numSuffix = baseName.substring(dashIndex+1);
            String basePrefix = baseName.substring(0,dashIndex);
            int numSuffixValue = new Integer(numSuffix).intValue();
            numSuffixValue++; 
            baseName = basePrefix + "-" + numSuffixValue;
        } else {
            baseName = baseName + "-1"; 
        }
        
        newName = baseName; 
        if (extension != null) {
            newName = newName + "." + extension; 
        }
        
        return newName;
    }
    
    /**
     *  Returns a content type string for a FileObject
     * 
     */
    private String getContentType(File fileObject){
        if (fileObject==null){
            return null;
        }
        String contentType;
        try {
            contentType = FileUtil.determineFileType(fileObject, fileObject.getName());
        } catch (IOException ex) {
            logger.info("FileUtil.determineFileType failed for file with name: " + fileObject.getName());
            contentType = null;
        }

        if ((contentType==null)||(contentType=="")){
            contentType = "MIME_TYPE_UNDETERMINED_DEFAULT";
        }
        return contentType;
        
    }
    /* 
     * This method creates a DataFile, and also saves the bytes from the suppplied 
     * InputStream in the temporary location. 
     * This method should only be called by the upper-level methods that handle 
     * file upload and creation for individual use cases - a single file upload, 
     * an upload of a zip archive that needs to be unpacked and turned into 
     * individual files, etc., and once the file name and mime type have already 
     * been figured out. 
    */
    
    private DataFile createSingleDataFile(DatasetVersion version, InputStream inputStream, String fileName, String contentType) {

        DataFile datafile = new DataFile(contentType);
        FileMetadata fmd = new FileMetadata();
        
        fmd.setLabel(checkForDuplicateFileNames(version,fileName));

        datafile.setOwner(version.getDataset());
        fmd.setDataFile(datafile);
        datafile.getFileMetadatas().add(fmd);
        if (version.getFileMetadatas() == null) {
            version.setFileMetadatas(new ArrayList());
        }
        version.getFileMetadatas().add(fmd);
        fmd.setDatasetVersion(version);
        version.getDataset().getFiles().add(datafile);

        // And save the file - but only if the InputStream is not null; 
        // (the temp file may be saved already - if this is a single
        // file upload case - and in that case this method gets called 
{'fraction_non_alphanumeric': 0.08660422074257984, 'fraction_numerical': 0.0018508069066696762, 'mean_word_length': 2.0315429353404038, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3436967', 'n_tokens_mistral': 11926, 'n_tokens_neox': 11437, 'n_words': 2980}