<commit_before>/* File:        $Id$
 * Revision:    $Revision$
 * Author:      $Author$
 * Date:        $Date$
 *
 * The Netarchive Suite - Software to harvest and preserve websites
 * Copyright 2004-2007 <NAME> and Statsbiblioteket, Denmark
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

package dk.netarkivet.common.distribute.arcrepository;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.archive.io.arc.ARCReader;
import org.archive.io.arc.ARCReaderFactory;
import org.archive.io.arc.ARCRecord;

import dk.netarkivet.common.distribute.RemoteFileFactory;
import dk.netarkivet.common.exceptions.ArgumentNotValid;
import dk.netarkivet.common.exceptions.IOFailure;
import dk.netarkivet.common.exceptions.IllegalState;
import dk.netarkivet.common.exceptions.PermissionDenied;
import dk.netarkivet.common.utils.FileUtils;
import dk.netarkivet.common.utils.MD5;
import dk.netarkivet.common.utils.Settings;
import dk.netarkivet.common.utils.batch.BatchLocalFiles;
import dk.netarkivet.common.utils.batch.FileBatchJob;

/**
 * A simple implementation of ArcRepositoryClient that just has a number of
 * local directories that it keeps its files in.  It doesn't implement
 * credentials checks or checksum storing.
 */
public class LocalArcRepositoryClient implements ArcRepositoryClient {
    /** The logger for this class. */
    private Log log = LogFactory.getLog(getClass());
    /** List of the directories that we store files in. Non-absolute dirs are
     * relative to the current directory. */
    private final List<File> storageDirs = new ArrayList<File>(1);
    /** If no directories are specified in the settings file, use a single
     * directory with this name.
     */
    private static final String DEFAULT_DIR_NAME = "ArcRepository";
    /** Store the file in the directories designated by this setting. */
    private static final String FILE_DIRS
            = "settings.common.arcrepositoryClient.fileDir";
    private static final String CREDENTIALS_SETTING
            = "settings.archive.bitarchive.thisCredentials";

    /** Create a new LocalArcRepositoryClient based on current settings. */
    public LocalArcRepositoryClient() {
        List<String> fileDirs =
                Arrays.asList(Settings.getAll(FILE_DIRS));
        if (fileDirs.size() == 0) {
            fileDirs.add(DEFAULT_DIR_NAME);
        }
        for (String fileName : fileDirs) {
            File f = new File(fileName);
            FileUtils.createDir(f);
            log.info("directory '" +  f.getAbsolutePath() 
                    + "' is part of this local archive repository");
            storageDirs.add(f);
        }
    }

    /** Call on shutdown to release external resources. */
    public void close() {
    }

    /**
     * Store the given file in the ArcRepository.  After storing, the file is
     * deleted.
     *
     * @param file A file to be stored. Must exist.
     * @throws IOFailure thrown if store is unsuccessful, or failed to clean
     * up files after the store operation.
     * @throws IllegalState if file already exists.
     * @throws ArgumentNotValid if file parameter is null or file is not an
     *                          existing file.
     */
    public void store(File file) throws IOFailure, ArgumentNotValid {
        ArgumentNotValid.checkNotNull(file, "File file");
        ArgumentNotValid.checkTrue(file.exists(), "File '" + file
                                                  + "' does not exist");
        if (findFile(file.getName()) != null) {
            throw new IllegalState("A file with the name '"
                                       + file.getName() + " is already stored");
        }
        for (File dir : storageDirs) {
            if (dir.canWrite() && FileUtils.getBytesFree(dir) > file.length()) {
                FileUtils.moveFile(file, new File(dir, file.getName()));
                return;
            }
        }
        throw new IOFailure("Not enough room for '" + file
                            + "' in any of the dirs " + storageDirs);
    }

    /**
     * Gets a single ARC record out of the ArcRepository.
     *
     * @param arcfile The name of a file containing the desired record.
     * @param index   The offset of the desired record in the file
     * @return a BitarchiveRecord-object, or null if request times out or object
     * is not found.
     * @throws ArgumentNotValid on null or empty filenames, or if index is
     * negative.
     * @throws IOFailure If the get operation failed.
     */
    public BitarchiveRecord get(String arcfile, long index)
            throws ArgumentNotValid {
        ArgumentNotValid.checkNotNullOrEmpty(arcfile, "String arcfile");
        ArgumentNotValid.checkNotNegative(index, "long index");
        File f = findFile(arcfile);
        if (f == null) {
            log.warn("File '" + arcfile 
                    + "' does not exist. Null BitarchiveRecord returned");
            return null;
        }
        ARCReader reader = null;
        ARCRecord record = null;
        try {
            reader = ARCReaderFactory.get(new File(f, arcfile), index);
            record = (ARCRecord) reader.get();
            return new BitarchiveRecord(record);
        } catch (IOException e) {
            throw new IOFailure("Error reading record from '"
                    + arcfile + "' offset " + index, e);
        } finally {
            if (record != null) {
                try {
                    record.close();
                } catch (IOException e) {
                    log.warn("Error closing ARC record '" + record + "'", e);
                }<commit_msg>[FROM NICOLAS @ BNF] Bug 1694: LocalArcRepositoryClient is broken<commit_after>/* File:        $Id$
 * Revision:    $Revision$
 * Author:      $Author$
 * Date:        $Date$
 *
 * The Netarchive Suite - Software to harvest and preserve websites
 * Copyright 2004-2007 Det Kongelige Bibliotek and Statsbiblioteket, Denmark
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

package dk.netarkivet.common.distribute.arcrepository;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.archive.io.arc.ARCReader;
import org.archive.io.arc.ARCReaderFactory;
import org.archive.io.arc.ARCRecord;

import dk.netarkivet.common.distribute.RemoteFileFactory;
import dk.netarkivet.common.exceptions.ArgumentNotValid;
import dk.netarkivet.common.exceptions.IOFailure;
import dk.netarkivet.common.exceptions.IllegalState;
import dk.netarkivet.common.exceptions.PermissionDenied;
import dk.netarkivet.common.utils.FileUtils;
import dk.netarkivet.common.utils.MD5;
import dk.netarkivet.common.utils.Settings;
import dk.netarkivet.common.utils.batch.BatchLocalFiles;
import dk.netarkivet.common.utils.batch.FileBatchJob;

/**
 * A simple implementation of ArcRepositoryClient that just has a number of
 * local directories that it keeps its files in.  It doesn't implement
 * credentials checks or checksum storing.
 */
public class LocalArcRepositoryClient implements ArcRepositoryClient {
    /** The logger for this class. */
    private Log log = LogFactory.getLog(getClass());
    /** List of the directories that we store files in. Non-absolute dirs are
     * relative to the current directory. */
    private final List<File> storageDirs = new ArrayList<File>(1);
    /** If no directories are specified in the settings file, use a single
     * directory with this name.
     */
    private static final String DEFAULT_DIR_NAME = "ArcRepository";
    /** Store the file in the directories designated by this setting. */
    private static final String FILE_DIRS
            = "settings.common.arcrepositoryClient.fileDir";
    private static final String CREDENTIALS_SETTING
            = "settings.archive.bitarchive.thisCredentials";

    /** Create a new LocalArcRepositoryClient based on current settings. */
    public LocalArcRepositoryClient() {
        List<String> fileDirs =
                Arrays.asList(Settings.getAll(FILE_DIRS));
        if (fileDirs.size() == 0) {
            fileDirs.add(DEFAULT_DIR_NAME);
        }
        for (String fileName : fileDirs) {
            File f = new File(fileName);
            FileUtils.createDir(f);
            log.info("directory '" +  f.getAbsolutePath() 
                    + "' is part of this local archive repository");
            storageDirs.add(f);
        }
    }

    /** Call on shutdown to release external resources. */
    public void close() {
    }

    /**
     * Store the given file in the ArcRepository.  After storing, the file is
     * deleted.
     *
     * @param file A file to be stored. Must exist.
     * @throws IOFailure thrown if store is unsuccessful, or failed to clean
     * up files after the store operation.
     * @throws IllegalState if file already exists.
     * @throws ArgumentNotValid if file parameter is null or file is not an
     *                          existing file.
     */
    public void store(File file) throws IOFailure, ArgumentNotValid {
        ArgumentNotValid.checkNotNull(file, "File file");
        ArgumentNotValid.checkTrue(file.exists(), "File '" + file
                                                  + "' does not exist");
        if (findFile(file.getName()) != null) {
            throw new IllegalState("A file with the name '"
                                       + file.getName() + " is already stored");
        }
        for (File dir : storageDirs) {
            if (dir.canWrite() && FileUtils.getBytesFree(dir) > file.length()) {
                FileUtils.moveFile(file, new File(dir, file.getName()));
                return;
            }
        }
        throw new IOFailure("Not enough room for '" + file
                            + "' in any of the dirs " + storageDirs);
    }

    /**
     * Gets a single ARC record out of the ArcRepository.
     *
     * @param arcfile The name of a file containing the desired record.
     * @param index   The offset of the desired record in the file
     * @return a BitarchiveRecord-object, or null if request times out or object
     * is not found.
     * @throws ArgumentNotValid on null or empty filenames, or if index is
     * negative.
     * @throws IOFailure If the get operation failed.
     */
    public BitarchiveRecord get(String arcfile, long index)
            throws ArgumentNotValid {
        ArgumentNotValid.checkNotNullOrEmpty(arcfile, "String arcfile");
        ArgumentNotValid.checkNotNegative(index, "long index");
        File f = findFile(arcfile);
        if (f == null) {
            log.warn("File '" + arcfile 
                    + "' does not exist. Null BitarchiveRecord returned");
            return null;
        }
        ARCReader reader = null;
        ARCRecord record = null;
        try {
//            reader = ARCReaderFactory.get(new File(f, arcfile), index);
        	reader = ARCReaderFactory.get(f, index);
            record = (ARCRecord) reader.get();
            return new BitarchiveRecord(record);
        } catch (IOException e) {
            throw new IOFailure("Error reading record from '"
                    + arcfile + "' offset " + index, e);
        } finally {
            if (record != null) {
                try {
                    record.close();
                } catch (IOException e) {
                    log.warn("Error closing ARC record '" + record + "'", e);
                }
{'fraction_non_alphanumeric': 0.0888855579704714, 'fraction_numerical': 0.003897174548452372, 'mean_word_length': 2.4795306388526726, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6964088', 'n_tokens_mistral': 3580, 'n_tokens_neox': 3522, 'n_words': 1210}