<commit_before>/* File:            $Id$
 * Revision:        $Revision$
 * Author:          $Author$
 * Date:            $Date$
 *
 * The Netarchive Suite - Software to harvest and preserve websites
 * Copyright 2004-2012 The Royal Danish Library, the Danish State and
 * University Library, the National Library of France and the Austrian
 * National Library.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
package dk.netarkivet.common.utils;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import org.apache.commons.io.IOUtils;

import dk.netarkivet.common.Constants;
import dk.netarkivet.common.exceptions.ArgumentNotValid;
import dk.netarkivet.common.exceptions.IOFailure;
import dk.netarkivet.common.exceptions.IllegalState;

/**
 * Calculates MD5 or SHA1 checksums on files using the built-in Java methods.
 */
public final class ChecksumCalculator {
    public static final String MD5 = "MD5";
    public static final String SHA1 = "SHA1";

    /**
     * Calculate MD5 for a file.
     *
     * @param src The file to calculate MD5 for.
     * @return The MD5 sum of a file as a 32 characters long Hex string.
     */
    public static String calculateMd5(final File src) {
        ArgumentNotValid.checkNotNull(src, "File src");
        ArgumentNotValid.checkTrue(src.isFile(), "Argument should be a file");
        // Get the MD5 and return it
        try {
            final FileInputStream fileInputStream = new FileInputStream(src);
            try {
                return calculateMd5(fileInputStream);
            } finally {
                IOUtils.closeQuietly(fileInputStream);
            }
        } catch (FileNotFoundException e) {
            throw new IOFailure("Could not read file '"
                                + src.getAbsolutePath() + "'", e);
        }
    }

    /**
     * Calculate the SHA-1 DIGEST for a file.
     *
     * @param src The file to calculate SHA-1 for.
     * @return The SHA-1 sum of a file as a 32 characters long Hex string.
     */
    public static String calculateSha1(final File src) {
        ArgumentNotValid.checkNotNull(src, "File src");
        ArgumentNotValid.checkTrue(src.isFile(), "Argument should be a file");
        // Get the SHA-1 digest and return it
        try {
            final FileInputStream fileInputStream = new FileInputStream(src);
            try {
                return calculateSha1(fileInputStream);
            } finally {
                IOUtils.closeQuietly(fileInputStream);
            }
        } catch (FileNotFoundException e) {
            throw new IOFailure("Could not read file '"
                                + src.getAbsolutePath() + "'", e);
        }
    }

    /**
     * Calculates an MD5 digest on an InputStream, throwing away the
     * data itself. Throws Alert if there is an error reading from the
     * stream
     *
     * @param instream An <code>InputStream</code> to calculate the MD5 digest on.
     * The contents of the stream will be consumed by this call, but the
     * stream will not be closed.
     * @return The calculated MD5 digest as a string.
     */
    public static String calculateMd5(final InputStream instream) {
        return calculateDigest(instream, "MD5");
    }

    /**
     * Calculates an SHA-1 digest on an InputStream, throwing away the 
     * data itself. Throws Alert if there is an error reading from the 
     * stream
     *
     * @param instream An <code>InputStream</code> to calculate the SHA-1 digest on.  
     * The contents of the stream will be consumed by this call, but the 
     * stream will not be closed.
     * @return The calculated SHA-1 digest as a string.
     */
    public static String calculateSha1(final InputStream instream) {
        return calculateDigest(instream, "SHA1");
    }

    /**
     * Generate an MD5 for a byte array.
     * @param msg The given bytearray
     * @return the MD5 for a byte array
     */
    public static String calculateMd5(final byte[] msg) {
        return toHex(getMessageDigest("MD5").digest(msg));
    }

    /**
     * Calculates a digest on an InputStream, throwing away the 
     * data itself. Throws Alert if there is an error reading from the 
     * stream
     *
     * @param instream An <code>InputStream</code> to calculate the digest on.  
     * The contents of the stream will be consumed by this call, but the 
     * stream will not be closed.
     * @param algorithm digest algorithm to use
     * @return The calculated digest as a string.
     */
    private static String calculateDigest(final InputStream instream,
                                          final String algorithm) {
        final byte[] buffer = new byte[Constants.IO_BUFFER_SIZE];
        final MessageDigest messageDigest = getMessageDigest(algorithm);
        messageDigest.reset();<commit_msg>NAS-2130 - Replace use of MD5 class with ChecksumCalculator class that also  supports sha1: Rework after CR-NAS-160.<commit_after>/* File:            $Id$
 * Revision:        $Revision$
 * Author:          $Author$
 * Date:            $Date$
 *
 * The Netarchive Suite - Software to harvest and preserve websites
 * Copyright 2004-2012 The Royal Danish Library, the Danish State and
 * University Library, the National Library of France and the Austrian
 * National Library.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
package dk.netarkivet.common.utils;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import org.apache.commons.io.IOUtils;

import dk.netarkivet.common.Constants;
import dk.netarkivet.common.exceptions.ArgumentNotValid;
import dk.netarkivet.common.exceptions.IOFailure;
import dk.netarkivet.common.exceptions.IllegalState;

/**
 * Calculates MD5 or SHA1 checksums on files using the built-in Java methods.
 */
public final class ChecksumCalculator {
    /** Defines the MD5 checksum algorithm */
    public static final String MD5 = "MD5";
    /** Defines the SHA1 checksum algorithm */
    public static final String SHA1 = "SHA1";

    /**
     * Calculate MD5 for a file.
     *
     * @param src The file to calculate MD5 for.
     * @return The MD5 sum of a file as a 32 characters long Hex string.
     */
    public static String calculateMd5(final File src) {
        ArgumentNotValid.checkNotNull(src, "File src");
        ArgumentNotValid.checkTrue(src.isFile(), "Argument should be a file");
        // Get the MD5 and return it
        try {
            final FileInputStream fileInputStream = new FileInputStream(src);
            try {
                return calculateMd5(fileInputStream);
            } finally {
                IOUtils.closeQuietly(fileInputStream);
            }
        } catch (FileNotFoundException e) {
            throw new IOFailure("Could not read file '"
                                + src.getAbsolutePath() + "'", e);
        }
    }

    /**
     * Calculate the SHA-1 DIGEST for a file.
     *
     * @param src The file to calculate SHA-1 for.
     * @return The SHA-1 sum of a file as a 32 characters long Hex string.
     */
    public static String calculateSha1(final File src) {
        ArgumentNotValid.checkNotNull(src, "File src");
        ArgumentNotValid.checkTrue(src.isFile(), "Argument should be a file");
        // Get the SHA-1 digest and return it
        try {
            final FileInputStream fileInputStream = new FileInputStream(src);
            try {
                return calculateSha1(fileInputStream);
            } finally {
                IOUtils.closeQuietly(fileInputStream);
            }
        } catch (FileNotFoundException e) {
            throw new IOFailure("Could not read file '"
                                + src.getAbsolutePath() + "'", e);
        }
    }

    /**
     * Calculates an MD5 digest on an InputStream, throwing away the
     * data itself. Throws Alert if there is an error reading from the
     * stream
     *
     * @param instream An <code>InputStream</code> to calculate the MD5 digest on.
     * The contents of the stream will be consumed by this call, but the
     * stream will not be closed.
     * @return The calculated MD5 digest as a string.
     */
    public static String calculateMd5(final InputStream instream) {
        return calculateDigest(instream, MD5);
    }

    /**
     * Calculates an SHA-1 digest on an InputStream, throwing away the 
     * data itself. Throws Alert if there is an error reading from the 
     * stream
     *
     * @param instream An <code>InputStream</code> to calculate the SHA-1 digest on.  
     * The contents of the stream will be consumed by this call, but the 
     * stream will not be closed.
     * @return The calculated SHA-1 digest as a string.
     */
    public static String calculateSha1(final InputStream instream) {
        return calculateDigest(instream, SHA1);
    }

    /**
     * Generate an MD5 for a byte array.
     * @param msg The given bytearray
     * @return the MD5 for a byte array
     */
    public static String calculateMd5(final byte[] msg) {
        return toHex(getMessageDigest(MD5).digest(msg));
    }

    /**
     * Calculates a digest on an InputStream, throwing away the 
     * data itself. Throws Alert if there is an error reading from the 
     * stream
     *
     * @param instream An <code>InputStream</code> to calculate the digest on.  
     * The contents of the stream will be consumed by this call, but the 
     * stream will not be closed.
     * @param algorithm digest algorithm to use
     * @return The calculated digest as a string.
     */
    private static String calculateDigest(final InputStream instream,
                                          final String algorithm) {
        final byte[] buffer = new byte[Constants.IO_BUFFER_SIZE];
        final MessageDigest messageDigest = getMessageDigest(algorithm);
        messageDigest.reset();
{'fraction_non_alphanumeric': 0.07546192367466759, 'fraction_numerical': 0.010792609221205318, 'mean_word_length': 2.4983388704318936, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3745763', 'n_tokens_mistral': 3107, 'n_tokens_neox': 3017, 'n_words': 1218}