<commit_before>package nl.bstoi.poiparser.core.util;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;

/**
 * <p>
 * Usefull class for reading certain excel properties and values
 * </p>
 * 
 * @author <NAME>
 * 
 */
public class PoiImportExportUtil {

	private static final Log log = LogFactory.getLog(PoiImportExportUtil.class);

	/**
	 * <p>
	 * Get sheet name from excel file
	 * </p>
	 * 
	 * @param excelFile
	 * @return
	 */
	public static List<String> getSheetNames(File excelFile) {
		FileInputStream fis = null;
		POIFSFileSystem poifs = null;
		try {
			log.debug("Loading sheet names from excel file:"
					+ excelFile.getName());
			fis = new FileInputStream(excelFile);
			poifs = new POIFSFileSystem(fis);
			HSSFWorkbook workbook = new HSSFWorkbook(poifs);
			List<String> sheetList = new ArrayList<String>();
			for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
				log.trace("Adding sheet name: " + workbook.getSheetName(i));
				sheetList.add(workbook.getSheetName(i));
			}
			fis.close();
			return sheetList;
		} catch (FileNotFoundException e) {
			log.error("File: " + excelFile.getName() + " cannot be found.");
		} catch (IOException e) {
			log.error("File: " + excelFile.getName() + " cannot be read.");
		}
		return null;
	}

	/**
	 * <p>
	 * Get excel files from a certain path
	 * </p>
	 * 
	 * @param path
	 * @return
	 */
	public static List<String> readExcelFiles(String path) {
		log.debug("Listing excel files from path: " + path);
		File directoryPath = new File(path);
		List<String> fileList = new ArrayList<String>();
		if (directoryPath.exists() && directoryPath.isDirectory()) {
			for (File file : directoryPath.listFiles()) {
				fileList.add(file.getName());
			}
		}
		return fileList;
	}

	/**
	 * <p>
	 * Get the file hash for a excel file
	 * </p>
	 * 
	 * @param excelFile
	 * @return
	 */
	public static String getFileHash(File excelFile) {
		try {
			MessageDigest md = MessageDigest.getInstance("SHA-256");
			FileInputStream fis = new FileInputStream(excelFile);

			byte[] dataBytes = new byte[1024];

			int nread = 0;
			while ((nread = fis.read(dataBytes)) != -1) {
				md.update(dataBytes, 0, nread);
			}
			;
			byte[] mdbytes = md.digest();

			// convert the byte to hex format method 2
			StringBuffer hexString = new StringBuffer();
			for (int i = 0; i < mdbytes.length; i++) {
				hexString.append(String.format("%02x", mdbytes[i]));
			}
			
			log.debug("Hash for file ["+excelFile.getName()+"] is: "+hexString.toString());
			fis.close();
			return hexString.toString();
		} catch (NoSuchAlgorithmException e) {
			log.error("Hashing algorithm cannot be found.");
		} catch (IOException e) {
			log.error("Error while reading file.");
		}
		return null;
	}
	
	public static String getByteArrayHash(byte[] bytes){
		try{
			MessageDigest md = MessageDigest.getInstance("SHA-256");
			byte[] dataBytes = new byte[1024];
			
			ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
	
			int nread = 0;
			while ((nread = bis.read(dataBytes)) != -1) {
				md.update(dataBytes, 0, nread);
			};
			
			byte[] mdbytes = md.digest();

			// convert the byte to hex format method 2
			StringBuffer hexString = new StringBuffer();
			for (int i = 0; i < mdbytes.length; i++) {
				hexString.append(String.format("%02x", mdbytes[i]));
			}
			
			bis.close();
			return hexString.toString();
			
			
		} catch (NoSuchAlgorithmException e) {
			log.error("Hashing algorithm cannot be found.");
		} catch (IOException e) {
			log.error("Error while reading file.");
		}
		return null;
	}

}<commit_msg>sheet names could now be determined from input stream<commit_after>package nl.bstoi.poiparser.core.util;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;

import java.io.*;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;

/**
 * <p>
 * Usefull class for reading certain excel properties and values
 * </p>
 *
 * @author <NAME>
 */
public class PoiImportExportUtil {

    private static final Log log = LogFactory.getLog(PoiImportExportUtil.class);

    /**
     * <p>
     * Get sheet name from excel file
     * </p>
     *
     * @param excelFile
     * @return
     */
    public static List<String> getSheetNames(final File excelFile) {
        try {
            log.debug("Loading sheet names from excel file:"
                    + excelFile.getName());
            return getSheetNames(new FileInputStream(excelFile));
        } catch (FileNotFoundException e) {
            log.error("File: " + excelFile.getName() + " cannot be found.");
        }
        return null;
    }


    public static List<String> getSheetNames(final InputStream inputStream) {
        POIFSFileSystem poifs = null;
        try {
            poifs = new POIFSFileSystem(inputStream);
            HSSFWorkbook workbook = new HSSFWorkbook(poifs);
            List<String> sheetList = new ArrayList<String>();
            for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
                log.trace("Adding sheet name: " + workbook.getSheetName(i));
                sheetList.add(workbook.getSheetName(i));
            }
            inputStream.close();
            return sheetList;
        } catch (IOException e) {
            log.error("Error while reading excel file");
        } finally {
            if (null != inputStream) {
                try {
                    inputStream.close();
                } catch (IOException e) {
                    log.error("Error while closing input stream");
                }
            }
        }
        return null;
    }

    /**
     * <p>
     * Get excel files from a certain path
     * </p>
     *
     * @param path
     * @return
     */
    public static List<String> readExcelFiles(String path) {
        log.debug("Listing excel files from path: " + path);
        File directoryPath = new File(path);
        List<String> fileList = new ArrayList<String>();
        if (directoryPath.exists() && directoryPath.isDirectory()) {
            for (File file : directoryPath.listFiles()) {
                fileList.add(file.getName());
            }
        }
        return fileList;
    }

    /**
     * <p>
     * Get the file hash for a excel file
     * </p>
     *
     * @param excelFile
     * @return
     */
    public static String getFileHash(File excelFile) {
        try {
            MessageDigest md = MessageDigest.getInstance("SHA-256");
            FileInputStream fis = new FileInputStream(excelFile);

            byte[] dataBytes = new byte[1024];

            int nread = 0;
            while ((nread = fis.read(dataBytes)) != -1) {
                md.update(dataBytes, 0, nread);
            }
            ;
            byte[] mdbytes = md.digest();

            // convert the byte to hex format method 2
            StringBuffer hexString = new StringBuffer();
            for (int i = 0; i < mdbytes.length; i++) {
                hexString.append(String.format("%02x", mdbytes[i]));
            }

            log.debug("Hash for file [" + excelFile.getName() + "] is: " + hexString.toString());
            fis.close();
            return hexString.toString();
        } catch (NoSuchAlgorithmException e) {
            log.error("Hashing algorithm cannot be found.");
        } catch (IOException e) {
            log.error("Error while reading file.");
        }
        return null;
    }

    public static String getByteArrayHash(byte[] bytes) {
        try {
            MessageDigest md = MessageDigest.getInstance("SHA-256");
            byte[] dataBytes = new byte[1024];

            ByteArrayInputStream bis = new ByteArrayInputStream(bytes);

            int nread = 0;
            while ((nread = bis.read(dataBytes)) != -1) {
                md.update(dataBytes, 0, nread);
            }
            ;

            byte[] mdbytes = md.digest();

            // convert the byte to hex format method 2
            StringBuffer hexString = new StringBuffer();
            for (int i = 0; i < mdbytes.length; i++) {
                hexString.append(String.format("%02x", mdbytes[i]));
            }

            bis.close();
            return hexString.toString();


        } catch (NoSuchAlgorithmException e) {
            log.error("Hashing algorithm cannot be found.");
        } catch (IOException e) {
            log.error("Error while reading file.");
        }
        return null;
    }

}
{'fraction_non_alphanumeric': 0.12526304131133015, 'fraction_numerical': 0.006423745708273342, 'mean_word_length': 2.3921863260706235, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7321529', 'n_tokens_mistral': 2920, 'n_tokens_neox': 2616, 'n_words': 686}