<commit_before>/*
 * The MIT License
 *
 * Copyright (c) 2013 The Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package org.broad.tribble;

import net.sf.samtools.seekablestream.SeekableStream;
import net.sf.samtools.seekablestream.SeekableStreamFactory;
import org.broad.tribble.index.Block;
import org.broad.tribble.index.Index;
import org.broad.tribble.index.IndexFactory;
import org.broad.tribble.readers.PositionalBufferedStream;
import org.broad.tribble.util.ParsingUtils;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.zip.GZIPInputStream;

/**
 * A reader for text feature files  (i.e. not tabix files).   This includes tribble-indexed and non-indexed files.  If
 * index both iterate() and query() methods are supported.
 * <p/>
 * Note: Non-indexed files can be gzipped, but not bgzipped.
 *
 * @author <NAME>
 * @since 2/11/12
 */
public class TribbleIndexedFeatureReader<T extends Feature, SOURCE> extends AbstractFeatureReader<T, SOURCE> {

    private Index index;

    /**
     * is the path pointing to our source data a regular file?
     */
    private final boolean pathIsRegularFile;

    /**
     * a potentially reusable seekable stream for queries over regular files
     */
    private SeekableStream seekableStream = null;

    /**
     * @param featurePath  - path to the feature file, can be a local file path, http url, or ftp url
     * @param codec        - codec to decode the features
     * @param requireIndex - true if the reader will be queries for specific ranges.  An index (idx) file must exist
     * @throws IOException
     */
    public TribbleIndexedFeatureReader(final String featurePath, final FeatureCodec<T, SOURCE> codec, final boolean requireIndex) throws IOException {

        super(featurePath, codec);

        if (requireIndex) {
            String indexFile = Tribble.indexFile(featurePath);
            if (ParsingUtils.resourceExists(indexFile)) {
                index = IndexFactory.loadIndex(indexFile);
            } else {
                // See if the index itself is gzipped
                indexFile = ParsingUtils.appendToPath(indexFile, ".gz");
                if (ParsingUtils.resourceExists(indexFile)) {
                    index = IndexFactory.loadIndex(indexFile);
                } else {
                    throw new TribbleException("An index is required, but none found.");
                }
            }
        }

        // does path point to a regular file?
        this.pathIsRegularFile = SeekableStreamFactory.isFilePath(path);

        readHeader();
    }

    /**
     * Get a seekable stream appropriate to read information from the current feature path
     * <p/>
     * This function ensures that if reuseStreamInQuery returns true then this function will only
     * ever return a single unique instance of SeekableStream for all calls given this instance of
     * TribbleIndexedFeatureReader.  If reuseStreamInQuery() returns false then the returned SeekableStream
     * will be newly opened each time, and should be closed after each use.
     *
     * @return a SeekableStream
     */
    private SeekableStream getSeekableStream() throws IOException {
        final SeekableStream result;
        if (reuseStreamInQuery()) {
            // if the stream points to an underlying file, only create the underlying seekable stream once
            if (seekableStream == null) seekableStream = SeekableStreamFactory.getInstance().getStreamFor(path);
            result = seekableStream;
        } else {
            // we are not reusing the stream, so make a fresh copy each time we request it
            result = SeekableStreamFactory.getInstance().getStreamFor(path);
        }

        return result;<commit_msg>Try loading index, even if one not required<commit_after>/*
 * The MIT License
 *
 * Copyright (c) 2013 The Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package org.broad.tribble;

import net.sf.samtools.seekablestream.SeekableStream;
import net.sf.samtools.seekablestream.SeekableStreamFactory;
import org.broad.tribble.index.Block;
import org.broad.tribble.index.Index;
import org.broad.tribble.index.IndexFactory;
import org.broad.tribble.readers.PositionalBufferedStream;
import org.broad.tribble.util.ParsingUtils;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.zip.GZIPInputStream;

/**
 * A reader for text feature files  (i.e. not tabix files).   This includes tribble-indexed and non-indexed files.  If
 * index both iterate() and query() methods are supported.
 * <p/>
 * Note: Non-indexed files can be gzipped, but not bgzipped.
 *
 * @author <NAME>
 * @since 2/11/12
 */
public class TribbleIndexedFeatureReader<T extends Feature, SOURCE> extends AbstractFeatureReader<T, SOURCE> {

    private Index index;

    /**
     * is the path pointing to our source data a regular file?
     */
    private final boolean pathIsRegularFile;

    /**
     * a potentially reusable seekable stream for queries over regular files
     */
    private SeekableStream seekableStream = null;

    /**
     * @param featurePath  - path to the feature file, can be a local file path, http url, or ftp url
     * @param codec        - codec to decode the features
     * @param requireIndex - true if the reader will be queries for specific ranges.  An index (idx) file must exist
     * @throws IOException
     */
    public TribbleIndexedFeatureReader(final String featurePath, final FeatureCodec<T, SOURCE> codec, final boolean requireIndex) throws IOException {

        super(featurePath, codec);

        String indexFile = Tribble.indexFile(featurePath);
        if (ParsingUtils.resourceExists(indexFile)) {
            index = IndexFactory.loadIndex(indexFile);
        } else {
            // See if the index itself is gzipped
            indexFile = ParsingUtils.appendToPath(indexFile, ".gz");
            if (ParsingUtils.resourceExists(indexFile)) {
                index = IndexFactory.loadIndex(indexFile);
            }
        }

        if (requireIndex && !this.hasIndex()) {
            throw new TribbleException("An index is required, but none found.");
        }

        // does path point to a regular file?
        this.pathIsRegularFile = SeekableStreamFactory.isFilePath(path);

        readHeader();
    }

    /**
     * Get a seekable stream appropriate to read information from the current feature path
     * <p/>
     * This function ensures that if reuseStreamInQuery returns true then this function will only
     * ever return a single unique instance of SeekableStream for all calls given this instance of
     * TribbleIndexedFeatureReader.  If reuseStreamInQuery() returns false then the returned SeekableStream
     * will be newly opened each time, and should be closed after each use.
     *
     * @return a SeekableStream
     */
    private SeekableStream getSeekableStream() throws IOException {
        final SeekableStream result;
        if (reuseStreamInQuery()) {
            // if the stream points to an underlying file, only create the underlying seekable stream once
            if (seekableStream == null) seekableStream = SeekableStreamFactory.getInstance().getStreamFor(path);
            result = seekableStream;
        } else {
            // we are not reusing the stream, so make a fresh copy each time we request it
            result = SeekableStreamFactory.getInstance().getStreamFor(path);
        }

        return result;
{'fraction_non_alphanumeric': 0.07208588957055215, 'fraction_numerical': 0.0018404907975460123, 'mean_word_length': 3.121786767804467, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5393926', 'n_tokens_mistral': 2574, 'n_tokens_neox': 2478, 'n_words': 1018}