<commit_before>/* filter_streams.cc
   <NAME>, 17 March 2005
   Copyright (c) 2005 <NAME>.  All rights reserved.
   
   This file is part of "Jeremy's Machine Learning Library", copyright (c)
   1999-2005 <NAME>.
   
   This program is available under the GNU General Public License, the terms
   of which are given by the file "license.txt" in the top level directory of
   the source code distribution.  If this file is missing, you have no right
   to use the program; please contact the author.

   This program is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   for more details.

   ---
   
   Implementation of filter streams.
*/

#include "filter_streams.h"
#include <fstream>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/bzip2.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/device/file.hpp>


using namespace std;


namespace ML {


/*****************************************************************************/
/* FILTER_OSTREAM                                                            */
/*****************************************************************************/

filter_ostream::filter_ostream()
    : ostream(std::cout.rdbuf())
{
}

filter_ostream::
filter_ostream(const std::string & file, std::ios_base::openmode mode)
    : ostream(std::cout.rdbuf())
{
    open(file, mode);
}

namespace {

bool ends_with(const std::string & str, const std::string & what)
{
    string::size_type result = str.rfind(what);
    return result != string::npos
        && result == str.size() - what.size();
}

} // file scope

void filter_ostream::
open(const std::string & file_, std::ios_base::openmode mode)
{
    using namespace boost::iostreams;

    string file = file_;
    if (file == "") file = "/dev/null";

    auto_ptr<filtering_ostream> new_stream
        (new filtering_ostream());

    bool gzip = (ends_with(file, ".gz") || ends_with(file, ".gz~"));
    bool bzip2 = (ends_with(file, ".bz2") || ends_with(file, ".bz2~"));

    if (gzip) new_stream->push(gzip_compressor());
    if (bzip2) new_stream->push(bzip2_compressor());
    
    if (file == "-") {
        new_stream->push(std::cout);
    }
    else {
        new_stream->push(file_sink(file.c_str(), mode));
    }

    stream.reset(new_stream.release());
    rdbuf(stream->rdbuf());

    //stream.reset(new ofstream(file.c_str(), mode));
}

void
filter_ostream::
close()
{
    rdbuf(0);
    //stream->close();
}


/*****************************************************************************/
/* FILTER_ISTREAM                                                            */
/*****************************************************************************/

filter_istream::filter_istream()
    : istream(std::cin.rdbuf())
{
}

filter_istream::
filter_istream(const std::string & file, std::ios_base::openmode mode)
    : istream(std::cin.rdbuf())
{
    open(file, mode);
}

void filter_istream::
open(const std::string & file_, std::ios_base::openmode mode)
{
    using namespace boost::iostreams;

    string file = file_;
    if (file == "") file = "/dev/null";
    
    auto_ptr<filtering_istream> new_stream
        (new filtering_istream());

    bool gzip = (ends_with(file, ".gz") || ends_with(file, ".gz~"));
    bool bzip2 = (ends_with(file, ".bz2") || ends_with(file, ".bz2~"));

    if (gzip) new_stream->push(gzip_decompressor());
    if (bzip2) new_stream->push(bzip2_decompressor());

    if (file == "-") {
        new_stream->push(std::cin);
    }
    else {
        new_stream->push(file_source(file.c_str(), mode));
    }
    
    stream.reset(new_stream.release());
    rdbuf(stream->rdbuf());
}
<commit_msg>filter_istream now throws if it can't open the file<commit_after>/* filter_streams.cc
   <NAME>, 17 March 2005
   Copyright (c) 2005 <NAME>.  All rights reserved.
   
   This file is part of "Jeremy's Machine Learning Library", copyright (c)
   1999-2005 <NAME>.
   
   This program is available under the GNU General Public License, the terms
   of which are given by the file "license.txt" in the top level directory of
   the source code distribution.  If this file is missing, you have no right
   to use the program; please contact the author.

   This program is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   for more details.

   ---
   
   Implementation of filter streams.
*/

#include "filter_streams.h"
#include <fstream>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/bzip2.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/device/file.hpp>
#include "jml/arch/exception.h"
#include <errno.h>


using namespace std;


namespace ML {


/*****************************************************************************/
/* FILTER_OSTREAM                                                            */
/*****************************************************************************/

filter_ostream::filter_ostream()
    : ostream(std::cout.rdbuf())
{
}

filter_ostream::
filter_ostream(const std::string & file, std::ios_base::openmode mode)
    : ostream(std::cout.rdbuf())
{
    open(file, mode);
}

namespace {

bool ends_with(const std::string & str, const std::string & what)
{
    string::size_type result = str.rfind(what);
    return result != string::npos
        && result == str.size() - what.size();
}

} // file scope

void filter_ostream::
open(const std::string & file_, std::ios_base::openmode mode)
{
    using namespace boost::iostreams;

    string file = file_;
    if (file == "") file = "/dev/null";

    auto_ptr<filtering_ostream> new_stream
        (new filtering_ostream());

    bool gzip = (ends_with(file, ".gz") || ends_with(file, ".gz~"));
    bool bzip2 = (ends_with(file, ".bz2") || ends_with(file, ".bz2~"));

    if (gzip) new_stream->push(gzip_compressor());
    if (bzip2) new_stream->push(bzip2_compressor());
    
    if (file == "-") {
        new_stream->push(std::cout);
    }
    else {
        new_stream->push(file_sink(file.c_str(), mode));
    }

    stream.reset(new_stream.release());
    rdbuf(stream->rdbuf());

    //stream.reset(new ofstream(file.c_str(), mode));
}

void
filter_ostream::
close()
{
    rdbuf(0);
    //stream->close();
}


/*****************************************************************************/
/* FILTER_ISTREAM                                                            */
/*****************************************************************************/

filter_istream::filter_istream()
    : istream(std::cin.rdbuf())
{
}

filter_istream::
filter_istream(const std::string & file, std::ios_base::openmode mode)
    : istream(std::cin.rdbuf())
{
    open(file, mode);
}

void filter_istream::
open(const std::string & file_, std::ios_base::openmode mode)
{
    using namespace boost::iostreams;

    string file = file_;
    if (file == "") file = "/dev/null";
    
    auto_ptr<filtering_istream> new_stream
        (new filtering_istream());

    bool gzip = (ends_with(file, ".gz") || ends_with(file, ".gz~"));
    bool bzip2 = (ends_with(file, ".bz2") || ends_with(file, ".bz2~"));

    if (gzip) new_stream->push(gzip_decompressor());
    if (bzip2) new_stream->push(bzip2_decompressor());

    if (file == "-") {
        new_stream->push(std::cin);
    }
    else {
        file_source source(file.c_str(), mode);
        if (!source.is_open())
            throw Exception("stream open failed for file %s: %s",
                            file_.c_str(), strerror(errno));
        new_stream->push(source);
    }

    stream.reset(new_stream.release());
    rdbuf(stream->rdbuf());
}

{'fraction_non_alphanumeric': 0.21194142893208787, 'fraction_numerical': 0.007573844988639233, 'mean_word_length': 3.3224222585924714, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5360902', 'n_tokens_mistral': 2479, 'n_tokens_neox': 2376, 'n_words': 580}