<commit_before>/*
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */
package dk.statsbiblioteket.summa.ingest.stream;

import de.schlichtherle.truezip.file.TFile;
import de.schlichtherle.truezip.file.TFileInputStream;
import de.schlichtherle.truezip.file.TVFS;
import de.schlichtherle.truezip.fs.FsSyncException;
import dk.statsbiblioteket.summa.common.Logging;
import dk.statsbiblioteket.summa.common.configuration.Configurable;
import dk.statsbiblioteket.summa.common.configuration.Configuration;
import dk.statsbiblioteket.summa.common.filter.Filter;
import dk.statsbiblioteket.summa.common.filter.Payload;
import dk.statsbiblioteket.summa.common.util.RecordUtil;
import dk.statsbiblioteket.util.qa.QAInfo;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.regex.Pattern;

/**
 * Traverses sub folders and archives in a given folder depth-first to deliver
 * files as streams. This is a plug-in replacement for FileReader and the
 * recommended way of feeding file content into Summa.
 * </p><p>
 * Note that the renaming of files only take place outside of archives.
 * The archives themselves are never changed.
 * </p><p>
 * Currently the TrueZIP package from http://truezip.java.net/ is used. This
 * package supports multiple parallel streams for ZIP and similar archives.
 * It supports regular ZIP files, ZIP64 (> 4GB ZIPs), tar and .tgz.
 * See http://java.net/projects/truezip for details.
 */
@QAInfo(level = QAInfo.Level.NORMAL,
        state = QAInfo.State.QA_NEEDED,
        author = "te")
public class ArchiveReader extends FileSystemReader {
    private static final Log log = LogFactory.getLog(ArchiveReader.class);

    /**
     * If true, the stream produced by {@link #next()} is copied and fully detached from the underlying file on
     * storage, before being passed on.
     * </p><p>
     * This is highly recommended as missing close on delivered Streams can lead to deadlocks.
     * </p><p>
     * Note: This has the unfortunate side-effect of premature marking of the source file as being completed.
     *       This does not change the recommendation of having the feature enabled.
     * </p><p>
     * Optional. Default is true.
     */
    public static final String CONF_COPY_STREAM = "archivereader.copystream";
    public static final boolean DEFAULT_COPY_STREAM = true;

    protected FileProvider provider;
    private final boolean copyStream;

    public ArchiveReader(Configuration conf) {
        boolean recursive = conf.getBoolean(CONF_RECURSIVE, DEFAULT_RECURSIVE);
        boolean reverseSort = conf.getBoolean(CONF_REVERSE_SORT, DEFAULT_REVERSE_SORT);
        Pattern filePattern = Pattern.compile(conf.getString(CONF_FILE_PATTERN, DEFAULT_FILE_PATTERN));
        String postfix = conf.getString(CONF_COMPLETED_POSTFIX, DEFAULT_COMPLETED_POSTFIX);
        String realPostfix = "".equals(postfix) ? null : postfix;
        copyStream = conf.getBoolean(CONF_COPY_STREAM, DEFAULT_COPY_STREAM);

        String rootString = conf.getString(CONF_ROOT_FOLDER);
        if ("".equals(rootString)) {
            throw new Configurable.ConfigurationException(
                    String.format("No root. This must be specified with %s", CONF_ROOT_FOLDER));<commit_msg>Adjustment: Changed default for copyStream to false, as some older setup deals with very large streams<commit_after>/*
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */
package dk.statsbiblioteket.summa.ingest.stream;

import de.schlichtherle.truezip.file.TFile;
import de.schlichtherle.truezip.file.TFileInputStream;
import de.schlichtherle.truezip.file.TVFS;
import de.schlichtherle.truezip.fs.FsSyncException;
import dk.statsbiblioteket.summa.common.Logging;
import dk.statsbiblioteket.summa.common.configuration.Configurable;
import dk.statsbiblioteket.summa.common.configuration.Configuration;
import dk.statsbiblioteket.summa.common.filter.Filter;
import dk.statsbiblioteket.summa.common.filter.Payload;
import dk.statsbiblioteket.summa.common.util.RecordUtil;
import dk.statsbiblioteket.util.qa.QAInfo;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.regex.Pattern;

/**
 * Traverses sub folders and archives in a given folder depth-first to deliver
 * files as streams. This is a plug-in replacement for FileReader and the
 * recommended way of feeding file content into Summa.
 * </p><p>
 * Note that the renaming of files only take place outside of archives.
 * The archives themselves are never changed.
 * </p><p>
 * Currently the TrueZIP package from http://truezip.java.net/ is used. This
 * package supports multiple parallel streams for ZIP and similar archives.
 * It supports regular ZIP files, ZIP64 (> 4GB ZIPs), tar and .tgz.
 * See http://java.net/projects/truezip for details.
 */
@QAInfo(level = QAInfo.Level.NORMAL,
        state = QAInfo.State.QA_NEEDED,
        author = "te")
public class ArchiveReader extends FileSystemReader {
    private static final Log log = LogFactory.getLog(ArchiveReader.class);

    /**
     * If true, the stream produced by {@link #next()} is copied and fully detached from the underlying file on
     * storage, before being passed on.
     * </p><p>
     * This is highly recommended as missing close on delivered Streams can lead to deadlocks. However, the default
     * is false to stay true to older setups, as lasge streams requires a similar amount of memory.
     * </p><p>
     * Note: This has the unfortunate side-effect of premature marking of the source file as being completed.
     *       This does not change the recommendation of having the feature enabled.
     * </p><p>
     * Optional. Default is false.
     */
    public static final String CONF_COPY_STREAM = "archivereader.copystream";
    public static final boolean DEFAULT_COPY_STREAM = false;

    protected FileProvider provider;
    private final boolean copyStream;

    public ArchiveReader(Configuration conf) {
        boolean recursive = conf.getBoolean(CONF_RECURSIVE, DEFAULT_RECURSIVE);
        boolean reverseSort = conf.getBoolean(CONF_REVERSE_SORT, DEFAULT_REVERSE_SORT);
        Pattern filePattern = Pattern.compile(conf.getString(CONF_FILE_PATTERN, DEFAULT_FILE_PATTERN));
        String postfix = conf.getString(CONF_COMPLETED_POSTFIX, DEFAULT_COMPLETED_POSTFIX);
        String realPostfix = "".equals(postfix) ? null : postfix;
        copyStream = conf.getBoolean(CONF_COPY_STREAM, DEFAULT_COPY_STREAM);

        String rootString = conf.getString(CONF_ROOT_FOLDER);
        if ("".equals(rootString)) {
            throw new Configurable.ConfigurationException(
                    String.format("No root. This must be specified with %s", CONF_ROOT_FOLDER));
{'fraction_non_alphanumeric': 0.08453505718547986, 'fraction_numerical': 0.00174042764793635, 'mean_word_length': 3.920489296636086, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6348740', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2282, 'n_words': 746}