<commit_before>/*
   Copyright (C) 2005-2012, by the President and Fellows of Harvard College.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   Dataverse Network - A web application to share, preserve and analyze research data.
   Developed at the Institute for Quantitative Social Science, Harvard University.
   Version 3.0.
*/
package edu.harvard.iq.dataverse.dataaccess;

import java.io.File;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.IOException;

import java.util.Iterator;

import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.Image;
import java.awt.Toolkit;
import java.awt.image.BufferedImage;
import java.awt.image.FilteredImageSource;
import java.awt.image.ImageFilter;
import java.awt.image.ImageProducer;
import java.awt.image.RGBImageFilter;

import javax.imageio.ImageIO;
import javax.imageio.ImageWriter;
import javax.imageio.stream.ImageOutputStream;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.util.FileUtil;
import java.util.logging.Logger;
import org.primefaces.util.Base64;

/**
 *
 * @author <NAME>
 */
public class ImageThumbConverter {
    public static int DEFAULT_CARDIMAGE_SIZE = 48;
    public static int DEFAULT_THUMBNAIL_SIZE = 64; 
    public static int DEFAULT_PREVIEW_SIZE = 400; 
    
    private static final Logger logger = Logger.getLogger(ImageThumbConverter.class.getCanonicalName());
    
    public ImageThumbConverter() {
    }
    
    public static boolean isThumbnailAvailable (DataFile file) {
        return isThumbnailAvailable(file, DEFAULT_THUMBNAIL_SIZE);
    }
    
    public static boolean isThumbnailAvailable(DataFile file, int size) {
        if (file == null) {
            return false;
        }

        String imageThumbFileName = null;

        try {

            DataFileIO dataAccess = file.getAccessObject();

            if (!dataAccess.isLocalFile()) {
                logger.fine("thumbnails are supported on local files only at this time.");
                return false;
            }

            logger.fine("Checking for thumbnail, file type: " + file.getContentType());

            if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
                imageThumbFileName = generateImageThumb(dataAccess.getFileSystemPath().toString(), size);
            } else if (file.getContentType().equalsIgnoreCase("application/pdf")) {
                imageThumbFileName = generatePDFThumb(dataAccess.getFileSystemPath().toString(), size);
            } else if (file.getContentType().equalsIgnoreCase("application/zipped-shapefile")) {
                imageThumbFileName = generateWorldMapThumb(dataAccess.getFileSystemPath().toString(), size);
            } else if (file.isTabularData() && file.hasGeospatialTag()) {
                imageThumbFileName = generateWorldMapThumb(dataAccess.getFileSystemPath().toString(), size);
            }
        } catch (IOException ioEx) {
            return false;
        }

        if (imageThumbFileName != null) {
            logger.fine("image thumb file name: " + imageThumbFileName);
            return true;
        }

        logger.fine("image thumb file name is null");
        return false;
    }
    
    public static FileAccessIO getImageThumb (FileAccessIO fileAccess) {
        return getImageThumb (fileAccess, DEFAULT_THUMBNAIL_SIZE);
    }
    
    public static FileAccessIO getImageThumb(FileAccessIO fileAccess, int size) {
        logger.fine("entering getImageThumb, size " + size);

        File imageThumbFile = getImageThumbAsFile(fileAccess, size);

        if (imageThumbFile != null) {

            fileAccess.closeInputStream();
            fileAccess.setSize(imageThumbFile.length());

            InputStream imageThumbInputStream = null;

            try {

                imageThumbInputStream = new FileInputStream(imageThumbFile);
            } catch (IOException ex) {
                return null;
            }

            if (imageThumbInputStream != null) {
                fileAccess.setInputStream(imageThumbInputStream);
                fileAccess.setIsLocalFile(true);

                fileAccess.setMimeType("image/png");
                logger.fine("getImageThumb: created fileAccess object for the image thumbnail file");
            } else {
                return null;
            }

            logger.fine("getImageThumb: returning file access object");
            return fileAccess;
        }
        logger.fine("returning null");
        return null;
    }
    
    public static String getImageThumbAsBase64(DataFile file, int size) {

        logger.fine("entering getImageThumb, size " + size);

        FileAccessIO fileAccess = null;

        try {

            fileAccess = (FileAccessIO) file.getAccessObject();
        } catch (IOException ex) {
            // too bad - but not fatal
            logger.warning("getImageThumbAsBase64: Failed to obtain FileAccess object for DataFile id " + file.getId());
            return null;
        }

        if (fileAccess != null && fileAccess.isLocalFile()) {
            try {
                fileAccess.open();
            } catch (IOException ex) {
                // too bad - but not fatal
                logger.warning("getImageThumbAsBase64: Failed to open FileAccess object for DataFile id " + file.getId());
                return null;
            }
            File imageThumbFile = getImageThumbAsFile(fileAccess, size);

            if (imageThumbFile != null) {

                return getImageAsBase64FromFile(imageThumbFile);
            }
        }

        return null;
    }
    
    /**
     * This method is suitable for returning a string to embed in an HTML img
     * tag (or JSF h:graphicImage tag) because the string begins with
     * "data:image/png;base64," but it is not suitable for returning a
     * downloadable image via an API call.
     */
    public static String getImageAsBase64FromFile(File imageFile) {
        InputStream imageThumbInputStream = null;
        try {

            int thumbSize = (int) imageFile.length();

            imageThumbInputStream = new FileInputStream(imageFile);

            if (imageThumbInputStream != null) {

                byte[] rawImageData = new byte[thumbSize];
                int bytesRead = imageThumbInputStream.read(rawImageData, 0, thumbSize);
                logger.fine("read " + bytesRead + " bytes of raw thumbnail image.");

                if (rawImageData != null) {
                    String imageDataBase64 = Base64.encodeToString(rawImageData, false);
                    //return FILE_CARD_IMAGE_URL + assignedThumbnailFileId;
                    return FileUtil.DATA_URI_SCHEME + imageDataBase64;
                }
            }
        } catch (IOException ex) {
            // too bad - but not fatal
            logger.warning("getImageAsBase64FromFile: Failed to read data from thumbnail file");
        } finally {
            if (imageThumbInputStream != null) {
                try {
                    imageThumbInputStream.close();
                } catch (Exception e) {
                }
            }
        }

        return null;
    }
    
    public static File getImageThumbAsFile(FileAccessIO fileAccess, int size) {
        String imageThumbFileName = null;
        try {
            if (fileAccess.getDataFile() != null && fileAccess.getDataFile().getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
                imageThumbFileName = generateImageThumb(fileAccess.getFileSystemPath().toString(), size);

            } else if (fileAccess.getDataFile() != null && fileAccess.getDataFile().getContentType().equalsIgnoreCase("application/pdf")) {
                imageThumbFileName = generatePDFThumb(fileAccess.getFileSystemPath().toString(), size);

            } else if (fileAccess.getDataFile() != null && fileAccess.getDataFile().getContentType().equalsIgnoreCase("application/zipped-shapefile")) {
                imageThumbFileName = generateWorldMapThumb(fileAccess.getFileSystemPath().toString(), size);
            
            } else if (fileAccess.getDataFile() != null && fileAccess.getDataFile().isTabularData() && fileAccess.getDataFile().hasGeospatialTag()) {
                imageThumbFileName = generateWorldMapThumb(fileAccess.getFileSystemPath().toString(), size);

            } else {
                return null;
            }

        } catch (IOException ioEx) {
            return null;
        }
        
        if (imageThumbFileName != null) {
            logger.fine("obtained non-null imageThumbFileName: "+imageThumbFileName);
            File imageThumbFile = new File(imageThumbFileName);
<commit_msg>be more liberal in showing thumbnails for tabular #3495<commit_after>/*
   Copyright (C) 2005-2012, by the President and Fellows of Harvard College.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   Dataverse Network - A web application to share, preserve and analyze research data.
   Developed at the Institute for Quantitative Social Science, Harvard University.
   Version 3.0.
*/
package edu.harvard.iq.dataverse.dataaccess;

import java.io.File;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.IOException;

import java.util.Iterator;

import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.Image;
import java.awt.Toolkit;
import java.awt.image.BufferedImage;
import java.awt.image.FilteredImageSource;
import java.awt.image.ImageFilter;
import java.awt.image.ImageProducer;
import java.awt.image.RGBImageFilter;

import javax.imageio.ImageIO;
import javax.imageio.ImageWriter;
import javax.imageio.stream.ImageOutputStream;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.util.FileUtil;
import java.util.logging.Logger;
import org.primefaces.util.Base64;

/**
 *
 * @author <NAME>
 */
public class ImageThumbConverter {
    public static int DEFAULT_CARDIMAGE_SIZE = 48;
    public static int DEFAULT_THUMBNAIL_SIZE = 64; 
    public static int DEFAULT_PREVIEW_SIZE = 400; 
    
    private static final Logger logger = Logger.getLogger(ImageThumbConverter.class.getCanonicalName());
    
    public ImageThumbConverter() {
    }
    
    public static boolean isThumbnailAvailable (DataFile file) {
        return isThumbnailAvailable(file, DEFAULT_THUMBNAIL_SIZE);
    }
    
    public static boolean isThumbnailAvailable(DataFile file, int size) {
        if (file == null) {
            return false;
        }

        String imageThumbFileName = null;

        try {

            DataFileIO dataAccess = file.getAccessObject();

            if (!dataAccess.isLocalFile()) {
                logger.fine("thumbnails are supported on local files only at this time.");
                return false;
            }

            logger.fine("Checking for thumbnail, file type: " + file.getContentType());

            if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
                imageThumbFileName = generateImageThumb(dataAccess.getFileSystemPath().toString(), size);
            } else if (file.getContentType().equalsIgnoreCase("application/pdf")) {
                imageThumbFileName = generatePDFThumb(dataAccess.getFileSystemPath().toString(), size);
            } else if (file.getContentType().equalsIgnoreCase("application/zipped-shapefile")) {
                imageThumbFileName = generateWorldMapThumb(dataAccess.getFileSystemPath().toString(), size);
            } else if (file.isTabularData() && file.hasGeospatialTag()) {
                imageThumbFileName = generateWorldMapThumb(dataAccess.getFileSystemPath().toString(), size);
            }
        } catch (IOException ioEx) {
            return false;
        }

        if (imageThumbFileName != null) {
            logger.fine("image thumb file name: " + imageThumbFileName);
            return true;
        }

        logger.fine("image thumb file name is null");
        return false;
    }
    
    public static FileAccessIO getImageThumb (FileAccessIO fileAccess) {
        return getImageThumb (fileAccess, DEFAULT_THUMBNAIL_SIZE);
    }
    
    public static FileAccessIO getImageThumb(FileAccessIO fileAccess, int size) {
        logger.fine("entering getImageThumb, size " + size);

        File imageThumbFile = getImageThumbAsFile(fileAccess, size);

        if (imageThumbFile != null) {

            fileAccess.closeInputStream();
            fileAccess.setSize(imageThumbFile.length());

            InputStream imageThumbInputStream = null;

            try {

                imageThumbInputStream = new FileInputStream(imageThumbFile);
            } catch (IOException ex) {
                return null;
            }

            if (imageThumbInputStream != null) {
                fileAccess.setInputStream(imageThumbInputStream);
                fileAccess.setIsLocalFile(true);

                fileAccess.setMimeType("image/png");
                logger.fine("getImageThumb: created fileAccess object for the image thumbnail file");
            } else {
                return null;
            }

            logger.fine("getImageThumb: returning file access object");
            return fileAccess;
        }
        logger.fine("returning null");
        return null;
    }
    
    public static String getImageThumbAsBase64(DataFile file, int size) {

        logger.fine("entering getImageThumb, size " + size);

        FileAccessIO fileAccess = null;

        try {

            fileAccess = (FileAccessIO) file.getAccessObject();
        } catch (IOException ex) {
            // too bad - but not fatal
            logger.warning("getImageThumbAsBase64: Failed to obtain FileAccess object for DataFile id " + file.getId());
            return null;
        }

        if (fileAccess != null && fileAccess.isLocalFile()) {
            try {
                fileAccess.open();
            } catch (IOException ex) {
                // too bad - but not fatal
                logger.warning("getImageThumbAsBase64: Failed to open FileAccess object for DataFile id " + file.getId());
                return null;
            }
            File imageThumbFile = getImageThumbAsFile(fileAccess, size);

            if (imageThumbFile != null) {

                return getImageAsBase64FromFile(imageThumbFile);
            }
        }

        return null;
    }
    
    /**
     * This method is suitable for returning a string to embed in an HTML img
     * tag (or JSF h:graphicImage tag) because the string begins with
     * "data:image/png;base64," but it is not suitable for returning a
     * downloadable image via an API call.
     */
    public static String getImageAsBase64FromFile(File imageFile) {
        InputStream imageThumbInputStream = null;
        try {

            int thumbSize = (int) imageFile.length();

            imageThumbInputStream = new FileInputStream(imageFile);

            if (imageThumbInputStream != null) {

                byte[] rawImageData = new byte[thumbSize];
                int bytesRead = imageThumbInputStream.read(rawImageData, 0, thumbSize);
                logger.fine("read " + bytesRead + " bytes of raw thumbnail image.");

                if (rawImageData != null) {
                    String imageDataBase64 = Base64.encodeToString(rawImageData, false);
                    //return FILE_CARD_IMAGE_URL + assignedThumbnailFileId;
                    return FileUtil.DATA_URI_SCHEME + imageDataBase64;
                }
            }
        } catch (IOException ex) {
            // too bad - but not fatal
            logger.warning("getImageAsBase64FromFile: Failed to read data from thumbnail file");
        } finally {
            if (imageThumbInputStream != null) {
                try {
                    imageThumbInputStream.close();
                } catch (Exception e) {
                }
            }
        }

        return null;
    }
    
    public static File getImageThumbAsFile(FileAccessIO fileAccess, int size) {
        String imageThumbFileName = null;
        try {
            if (fileAccess.getDataFile() != null && fileAccess.getDataFile().getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
                imageThumbFileName = generateImageThumb(fileAccess.getFileSystemPath().toString(), size);

            } else if (fileAccess.getDataFile() != null && fileAccess.getDataFile().getContentType().equalsIgnoreCase("application/pdf")) {
                imageThumbFileName = generatePDFThumb(fileAccess.getFileSystemPath().toString(), size);

            } else if (fileAccess.getDataFile() != null && fileAccess.getDataFile().getContentType().equalsIgnoreCase("application/zipped-shapefile")) {
                imageThumbFileName = generateWorldMapThumb(fileAccess.getFileSystemPath().toString(), size);
            
            } else if (fileAccess.getDataFile() != null && fileAccess.getDataFile().isTabularData()) {
                imageThumbFileName = generateWorldMapThumb(fileAccess.getFileSystemPath().toString(), size);

            } else {
                return null;
            }

        } catch (IOException ioEx) {
            return null;
        }
        
        if (imageThumbFileName != null) {
            logger.fine("obtained non-null imageThumbFileName: "+imageThumbFileName);
            File imageThumbFile = new File(imageThumbFileName);

{'fraction_non_alphanumeric': 0.09001903725863476, 'fraction_numerical': 0.005548001087843351, 'mean_word_length': 2.381644289130035, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3753097', 'n_tokens_mistral': 4926, 'n_tokens_neox': 4687, 'n_words': 1217}