<commit_before>/* File:        $Id$
 * Revision:    $Revision$
 * Date:        $Date$
 * Author:      $Author$
 *
 * The Netarchive Suite - Software to harvest and preserve websites
 * Copyright 2004-2007 <NAME> and Statsbiblioteket, Denmark
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
package dk.netarkivet.archive.arcrepository.bitpreservation;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import dk.netarkivet.archive.Constants;
import dk.netarkivet.archive.arcrepositoryadmin.AdminData;
import dk.netarkivet.archive.arcrepositoryadmin.ArcRepositoryEntry;
import dk.netarkivet.archive.arcrepositoryadmin.ReadOnlyAdminData;
import dk.netarkivet.common.distribute.arcrepository.ArcRepositoryClientFactory;
import dk.netarkivet.common.distribute.arcrepository.BatchStatus;
import dk.netarkivet.common.distribute.arcrepository.BitArchiveStoreState;
import dk.netarkivet.common.distribute.arcrepository.Location;
import dk.netarkivet.common.distribute.arcrepository.PreservationArcRepositoryClient;
import dk.netarkivet.common.exceptions.ArgumentNotValid;
import dk.netarkivet.common.exceptions.IOFailure;
import dk.netarkivet.common.exceptions.IllegalState;
import dk.netarkivet.common.exceptions.NetarkivetException;
import dk.netarkivet.common.exceptions.NotImplementedException;
import dk.netarkivet.common.exceptions.PermissionDenied;
import dk.netarkivet.common.utils.CleanupHook;
import dk.netarkivet.common.utils.CleanupIF;
import dk.netarkivet.common.utils.FileUtils;
import dk.netarkivet.common.utils.I18n;
import dk.netarkivet.common.utils.KeyValuePair;
import dk.netarkivet.common.utils.arc.FileBatchJob;

/**
 * Class handling integrity check of a given bit archive. <p/> This class must
 * run on the same machine as the arcrepository, as it uses the same admin data
 * file (read-only).  However, it still talks JMS with the arcrepository.
 */
public class FileBasedActiveBitPreservation
        implements ActiveBitPreservation, CleanupIF {
    /** the class log. */
    private static Log log = LogFactory.getLog(FileBasedActiveBitPreservation.class);

    /** Internationalisation object. */
    private static final I18n I18N
            = new I18n(Constants.TRANSLATIONS_BUNDLE);

    /**
     * This should be updated at the entrance of each major use block, to ensure
     * it is reasonably in sync with the file.  We cannot, however, guarantee
     * total sync, as the file can change at any time.  We consider it good
     * enough that it is updated every time there is user interaction.
     */
    private ReadOnlyAdminData admin;

    /**
     * Note that having this as a singleton prevents more than one user from
     * working on this at the same time.  We have no safeguards against abuse!
     */
    private static FileBasedActiveBitPreservation instance = null;

    /**
     * Every time we need to store intermediate data during preservation, files
     * will be put in a temp-dir with this prefix.
     */
    private static final String TMP_DIR_PREFIX = "bitpreservation";
    /** Hook to close down application. */
    private CleanupHook closeHook;

    /** Initalises an ActiveBitPreservation instance. */
    protected FileBasedActiveBitPreservation() {
        this.admin = AdminData.getReadOnlyInstance();
        closeHook = new CleanupHook(this);
        Runtime.getRuntime().addShutdownHook(closeHook);
    }

    /**
     * Get singleton ActiveBitPreservation instance.
     *
     * @return a singleton ActiveBitPreservation instance
     */
    public static FileBasedActiveBitPreservation getInstance() {
        synchronized (FileBasedActiveBitPreservation.class) {
            if (instance == null) {
                instance = new FileBasedActiveBitPreservation();
            }
        }
        return instance;
    }

    /**
     * Retrieve the preservation status for the file with a given filename.
     *
     * @param filename a given filename
     *
     * @return the preservation status for the file with a given filename, or
     *         null if the file named does not exist.
     *
     * @throws ArgumentNotValid if argument is null or the empty string
     */
    public FilePreservationStatus getFilePreservationStatus(String filename) {
        ArgumentNotValid.checkNotNullOrEmpty(filename, "filename");
        // Start by retrieving the admin status
        admin.synchronize();
        ArcRepositoryEntry ae = admin.getEntry(filename);

        // create and return the preservation status
        if (ae != null) {
            return new FilePreservationStatus(filename, ae, getChecksumMap(
                    filename));
        } else {
            return null;
        }
    }

    /** Generate a map of checksums for this file in the bitarchive.
     *
     * @return Map containing the output of checksum jobs from the bitarchives.
     * @param filename
     */
    private Map<Location, List<String>> getChecksumMap(String filename) {
        // get the checksum information
        Map<Location, List<String>> baname2checksum =
                new HashMap<Location, List<String>>();
        for (Location ba : Location.getKnown()) {
            List<String> checksum = getChecksums(ba, filename);
            log.debug("Putting checksum line '" + checksum + "' in for " + ba);
            baname2checksum.put(ba, checksum);
        }
        return baname2checksum;
    }

    /**
     * Get the checksum of a single file in a bitarchive.
     *
     * Note that this method runs a batch job on the bitarchives, and therefore
     * takes a long time.
     *
     * @param ba The bitarchive to ask for checksum
     * @param filename
     * @return The MD5 checksums of the file, or the empty string if the file was
     *         not in the bitarchive.
     * @see ChecksumJob#parseLine(String)
     */
    private List<String> getChecksums(Location ba, String filename) {
        ChecksumJob checksumJob = new ChecksumJob();
        checksumJob.processOnlyFileNamed(filename);
        String res;
        try {
            PreservationArcRepositoryClient arcrep =
                    ArcRepositoryClientFactory.getPreservationInstance();<commit_msg>Fix compileerrors in FilePreservationStatusTester<commit_after>/* File:        $Id$
 * Revision:    $Revision$
 * Date:        $Date$
 * Author:      $Author$
 *
 * The Netarchive Suite - Software to harvest and preserve websites
 * Copyright 2004-2007 <NAME> and Statsbiblioteket, Denmark
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
package dk.netarkivet.archive.arcrepository.bitpreservation;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import dk.netarkivet.archive.Constants;
import dk.netarkivet.archive.arcrepositoryadmin.AdminData;
import dk.netarkivet.archive.arcrepositoryadmin.ArcRepositoryEntry;
import dk.netarkivet.archive.arcrepositoryadmin.ReadOnlyAdminData;
import dk.netarkivet.common.distribute.arcrepository.ArcRepositoryClientFactory;
import dk.netarkivet.common.distribute.arcrepository.BatchStatus;
import dk.netarkivet.common.distribute.arcrepository.BitArchiveStoreState;
import dk.netarkivet.common.distribute.arcrepository.Location;
import dk.netarkivet.common.distribute.arcrepository.PreservationArcRepositoryClient;
import dk.netarkivet.common.exceptions.ArgumentNotValid;
import dk.netarkivet.common.exceptions.IOFailure;
import dk.netarkivet.common.exceptions.IllegalState;
import dk.netarkivet.common.exceptions.NetarkivetException;
import dk.netarkivet.common.exceptions.NotImplementedException;
import dk.netarkivet.common.exceptions.PermissionDenied;
import dk.netarkivet.common.utils.CleanupHook;
import dk.netarkivet.common.utils.CleanupIF;
import dk.netarkivet.common.utils.FileUtils;
import dk.netarkivet.common.utils.I18n;
import dk.netarkivet.common.utils.KeyValuePair;
import dk.netarkivet.common.utils.arc.FileBatchJob;

/**
 * Class handling integrity check of a given bit archive. <p/> This class must
 * run on the same machine as the arcrepository, as it uses the same admin data
 * file (read-only).  However, it still talks JMS with the arcrepository.
 */
public class FileBasedActiveBitPreservation
        implements ActiveBitPreservation, CleanupIF {
    /** the class log. */
    private static Log log = LogFactory.getLog(FileBasedActiveBitPreservation.class);

    /** Internationalisation object. */
    private static final I18n I18N
            = new I18n(Constants.TRANSLATIONS_BUNDLE);

    /**
     * This should be updated at the entrance of each major use block, to ensure
     * it is reasonably in sync with the file.  We cannot, however, guarantee
     * total sync, as the file can change at any time.  We consider it good
     * enough that it is updated every time there is user interaction.
     */
    private ReadOnlyAdminData admin;

    /**
     * Note that having this as a singleton prevents more than one user from
     * working on this at the same time.  We have no safeguards against abuse!
     */
    private static FileBasedActiveBitPreservation instance = null;

    /**
     * Every time we need to store intermediate data during preservation, files
     * will be put in a temp-dir with this prefix.
     */
    private static final String TMP_DIR_PREFIX = "bitpreservation";
    /** Hook to close down application. */
    private CleanupHook closeHook;

    /** Initalises an ActiveBitPreservation instance. */
    protected FileBasedActiveBitPreservation() {
        this.admin = AdminData.getReadOnlyInstance();
        closeHook = new CleanupHook(this);
        Runtime.getRuntime().addShutdownHook(closeHook);
    }

    /**
     * Get singleton ActiveBitPreservation instance.
     *
     * @return a singleton ActiveBitPreservation instance
     */
    public static FileBasedActiveBitPreservation getInstance() {
        synchronized (FileBasedActiveBitPreservation.class) {
            if (instance == null) {
                instance = new FileBasedActiveBitPreservation();
            }
        }
        return instance;
    }

    /**
     * Retrieve the preservation status for the file with a given filename.
     *
     * @param filename a given filename
     *
     * @return the preservation status for the file with a given filename, or
     *         null if the file named does not exist.
     *
     * @throws ArgumentNotValid if argument is null or the empty string
     */
    public FilePreservationStatus getFilePreservationStatus(String filename) {
        ArgumentNotValid.checkNotNullOrEmpty(filename, "filename");
        // Start by retrieving the admin status
        admin.synchronize();
        ArcRepositoryEntry ae = admin.getEntry(filename);

        // create and return the preservation status
        if (ae != null) {
            return new FilePreservationStatus(filename, ae, getChecksumMap(
                    filename));
        } else {
            return null;
        }
    }

    /** Generate a map of checksums for this file in the bitarchive.
     *
     * @return Map containing the output of checksum jobs from the bitarchives.
     * @param filename
     */
    static Map<Location, List<String>> getChecksumMap(String filename) {
        // get the checksum information
        Map<Location, List<String>> baname2checksum =
                new HashMap<Location, List<String>>();
        for (Location ba : Location.getKnown()) {
            List<String> checksum = getChecksums(ba, filename);
            log.debug("Putting checksum line '" + checksum + "' in for " + ba);
            baname2checksum.put(ba, checksum);
        }
        return baname2checksum;
    }

    /**
     * Get the checksum of a single file in a bitarchive.
     *
     * Note that this method runs a batch job on the bitarchives, and therefore
     * takes a long time.
     *
     * @param ba The bitarchive to ask for checksum
     * @param filename
     * @return The MD5 checksums of the file, or the empty string if the file was
     *         not in the bitarchive.
     * @see ChecksumJob#parseLine(String)
     */
    static List<String> getChecksums(Location ba, String filename) {
        ChecksumJob checksumJob = new ChecksumJob();
        checksumJob.processOnlyFileNamed(filename);
        String res;
        try {
            PreservationArcRepositoryClient arcrep =
                    ArcRepositoryClientFactory.getPreservationInstance();
{'fraction_non_alphanumeric': 0.07663421418636995, 'fraction_numerical': 0.004589707927677329, 'mean_word_length': 3.173244341265235, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4268816', 'n_tokens_mistral': 3968, 'n_tokens_neox': 3913, 'n_words': 1280}