<commit_before>/*
 *  Copyright 2008-2009 NVIDIA Corporation
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */


#pragma once

#include <cusp/coo_matrix.h>
#include <cusp/exception.h>

#include <thrust/sort.h>
#include <thrust/iterator/zip_iterator.h>

#include <vector>
#include <string>
#include <sstream>
#include <iostream>
#include <fstream>

namespace cusp
{
namespace io
{

namespace detail
{

inline
void tokenize(std::vector<std::string>& tokens,
              const std::string& str,
              const std::string& delimiters = "\n\r\t ")
{
    // Skip delimiters at beginning.
    std::string::size_type lastPos = str.find_first_not_of(delimiters, 0);
    // Find first "non-delimiter".
    std::string::size_type pos     = str.find_first_of(delimiters, lastPos);

    while (std::string::npos != pos || std::string::npos != lastPos)
    {
        // Found a token, add it to the vector.
        tokens.push_back(str.substr(lastPos, pos - lastPos));
        // Skip delimiters.  Note the "not_of"
        lastPos = str.find_first_not_of(delimiters, pos);
        // Find next "non-delimiter"
        pos = str.find_first_of(delimiters, lastPos);
    }
}

} // end namespace detail




struct matrix_market_banner
{
    std::string storage;    // "array" or "coordinate"
    std::string symmetry;   // "general", "symmetric", "hermitian", or "skew-symmetric" 
    std::string type;       // "complex", "real", "integer", or "pattern"
};

inline void read_matrix_market_banner(matrix_market_banner& banner, std::istream& file)
{
    std::string line;
    std::vector<std::string> tokens;

    // read first line
    std::getline(file, line);
    detail::tokenize(tokens, line); 

    if (tokens.size() != 5 || tokens[0] != "%%MatrixMarket" || tokens[1] != "matrix")
        throw cusp::io_exception("invalid MatrixMarket banner");

    banner.storage  = tokens[2];
    banner.type     = tokens[3];
    banner.symmetry = tokens[4];

    if (banner.storage != "array" && banner.storage != "coordinate")
        throw cusp::io_exception("invalid MatrixMarket storage format [" + banner.storage + "]");
    
    if (banner.type != "complex" && banner.type != "real" 
            && banner.type != "integer" && banner.type != "pattern")
        throw cusp::io_exception("invalid MatrixMarket data type [" + banner.type + "]");

    if (banner.symmetry != "general" && banner.symmetry != "symmetric" 
            && banner.symmetry != "hermitian" && banner.symmetry != "skew-symmetric")
        throw cusp::io_exception("invalid MatrixMarket symmetry [" + banner.symmetry + "]");
}

inline void read_matrix_market_banner(matrix_market_banner& banner, const std::string& filename)
{
    std::ifstream file(filename.c_str());

    if (!file)
        throw cusp::io_exception(std::string("invalid file: [") + filename + std::string("]"));

    read_matrix_market_banner(banner, file);
}

template <typename IndexType, typename ValueType>
void read_matrix_market_file(cusp::coo_matrix<IndexType,ValueType,cusp::host_memory>& coo, const std::string& filename)
{
    std::ifstream file(filename.c_str());

    if (!file)
        throw cusp::io_exception(std::string("invalid file name: [") + filename + std::string("]"));

    read_matrix_market_stream(coo, file);
}

template <typename IndexType, typename ValueType>
void read_matrix_market_stream(cusp::coo_matrix<IndexType,ValueType,cusp::host_memory>& coo, std::istream& file)
{
    // read banner
    matrix_market_banner banner;
    read_matrix_market_banner(banner, file);
   
    // read file contents line by line
    std::string line;
    
    // skip over banner and comments
    do
    {
        std::getline(file, line);
    } while (line[0] == '%');<commit_msg>Added a type check when reading Matrix Market files.<commit_after>/*
 *  Copyright 2008-2009 NVIDIA Corporation
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */


#pragma once

#include <cusp/coo_matrix.h>
#include <cusp/exception.h>

#include <thrust/sort.h>
#include <thrust/iterator/zip_iterator.h>

#include <vector>
#include <string>
#include <sstream>
#include <iostream>
#include <fstream>

namespace cusp
{
namespace io
{

namespace detail
{

inline
void tokenize(std::vector<std::string>& tokens,
              const std::string& str,
              const std::string& delimiters = "\n\r\t ")
{
    // Skip delimiters at beginning.
    std::string::size_type lastPos = str.find_first_not_of(delimiters, 0);
    // Find first "non-delimiter".
    std::string::size_type pos     = str.find_first_of(delimiters, lastPos);

    while (std::string::npos != pos || std::string::npos != lastPos)
    {
        // Found a token, add it to the vector.
        tokens.push_back(str.substr(lastPos, pos - lastPos));
        // Skip delimiters.  Note the "not_of"
        lastPos = str.find_first_not_of(delimiters, pos);
        // Find next "non-delimiter"
        pos = str.find_first_of(delimiters, lastPos);
    }
}

template<typename I, typename M>
void check_matrix_market_type(const cusp::coo_matrix<I,float,M>& coo, const std::string& type)
{
    if( type=="complex" )
        throw cusp::io_exception("complex-valued matrices incompatible with 'float' containers");
    // else: integer, real, and pattern are all allowed
}

template<typename I, typename M>
void check_matrix_market_type(const cusp::coo_matrix<I,double,M>& coo, const std::string& type)
{
    if( type=="complex" )
        throw cusp::io_exception("complex-valued matrices incompatible with 'double' containers");
    // else: integer, real, and pattern are all allowed
}

// TODO: Add ValueType=complex case here when complex data is supported

template<typename I, typename V, typename M>
void check_matrix_market_type(const cusp::coo_matrix<I,V,M>& coo, const std::string& type)
{
    if( type=="real" )
        throw cusp::io_exception("real-valued matrices require a container with 'float' or 'double' values");
}

} // end namespace detail




struct matrix_market_banner
{
    std::string storage;    // "array" or "coordinate"
    std::string symmetry;   // "general", "symmetric", "hermitian", or "skew-symmetric" 
    std::string type;       // "complex", "real", "integer", or "pattern"
};

inline void read_matrix_market_banner(matrix_market_banner& banner, std::istream& file)
{
    std::string line;
    std::vector<std::string> tokens;

    // read first line
    std::getline(file, line);
    detail::tokenize(tokens, line); 

    if (tokens.size() != 5 || tokens[0] != "%%MatrixMarket" || tokens[1] != "matrix")
        throw cusp::io_exception("invalid MatrixMarket banner");

    banner.storage  = tokens[2];
    banner.type     = tokens[3];
    banner.symmetry = tokens[4];

    if (banner.storage != "array" && banner.storage != "coordinate")
        throw cusp::io_exception("invalid MatrixMarket storage format [" + banner.storage + "]");
    
    if (banner.type != "complex" && banner.type != "real" 
            && banner.type != "integer" && banner.type != "pattern")
        throw cusp::io_exception("invalid MatrixMarket data type [" + banner.type + "]");

    if (banner.symmetry != "general" && banner.symmetry != "symmetric" 
            && banner.symmetry != "hermitian" && banner.symmetry != "skew-symmetric")
        throw cusp::io_exception("invalid MatrixMarket symmetry [" + banner.symmetry + "]");
}

inline void read_matrix_market_banner(matrix_market_banner& banner, const std::string& filename)
{
    std::ifstream file(filename.c_str());

    if (!file)
        throw cusp::io_exception(std::string("invalid file: [") + filename + std::string("]"));

    read_matrix_market_banner(banner, file);
}

template <typename IndexType, typename ValueType>
void read_matrix_market_file(cusp::coo_matrix<IndexType,ValueType,cusp::host_memory>& coo, const std::string& filename)
{
    std::ifstream file(filename.c_str());

    if (!file)
        throw cusp::io_exception(std::string("invalid file name: [") + filename + std::string("]"));

    read_matrix_market_stream(coo, file);
}

template <typename IndexType, typename ValueType>
void read_matrix_market_stream(cusp::coo_matrix<IndexType,ValueType,cusp::host_memory>& coo, std::istream& file)
{
    // read banner
    matrix_market_banner banner;
    read_matrix_market_banner(banner, file);

    // check for type mismatch with container
    detail::check_matrix_market_type(coo, banner.type);
   
    // read file contents line by line
    std::string line;
    
    // skip over banner and comments
    do
    {
        std::getline(file, line);
    } while (line[0] == '%');
{'fraction_non_alphanumeric': 0.13563026959508692, 'fraction_numerical': 0.0041636306859581555, 'mean_word_length': 3.7144259077526987, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4310382', 'n_tokens_mistral': 3111, 'n_tokens_neox': 2958, 'n_words': 784}