<commit_before>#include "yacas/platfileio.h"

#ifdef _WIN32
#define MAP_TO_WIN32_PATH_SEPARATOR
#endif // WIN32

static void MapPathSeparators(std::string& filename)
{
#ifdef MAP_TO_WIN32_PATH_SEPARATOR
    for (std::size_t i = 0; i < filename.size(); ++i)
        if (filename[i] == '/')
            filename[i] = '\\';
#endif
}


StdFileInput::StdFileInput(std::istream& stream, InputStatus& aStatus):
    LispInput(aStatus),
    _stream(stream)
{
}

StdFileInput::StdFileInput(LispLocalFile& file, InputStatus& aStatus):
    LispInput(aStatus),
    _stream(file.stream),
    _position(0),
    _cp_ready(false)
{
}

char32_t StdFileInput::Next()
{
    if (!_cp_ready)
        _get();

    if (EndOfStream())
        return std::char_traits<char32_t>::eof();
    
    _cp_ready = false;
    _position += 1;
    
    return _cp;
}

char32_t StdFileInput::Peek()
{
    if (EndOfStream())
        return std::char_traits<char32_t>::eof();

    if (!_cp_ready)
        _get();
    
    return _cp;
}

void StdFileInput::Rewind()
{
    _stream.seekg(0);
    _position = 0;
    _cp_ready = false;
}

bool StdFileInput::EndOfStream() const
{
    if (_stream.eof())
        return true;
    
    if (!_cp_ready)
        _get();
            
    return _stream.eof();
}

std::size_t StdFileInput::Position() const
{
    return _position;
}

void StdFileInput::SetPosition(std::size_t n)
{
    Rewind();
    
    for (std::size_t i = 0; i < n; ++i)
        Next();
}

void StdFileInput::_get() const
{
    char p[4];
    char* q = p;
    
    *q++ = _stream.get();
    
    while (!_stream.eof() && !utf8::is_valid(p, q))
        *q++ = _stream.get();
    
    if (_stream.eof())
        return;
    
    utf8::utf8to32(p, q, &_cp);
    _cp_ready = true;
}


std::string InternalFindFile(const std::string& fname, const std::vector<std::string>& dirs)
{
    std::string path(fname);

    MapPathSeparators(path);

    std::ifstream f(path);
    for (std::size_t i = 0; !f.good() && i < dirs.size(); ++i) {
        path = dirs[i] + fname;
        MapPathSeparators(path);
        f = std::ifstream(path);
    }

    if (!f.good())
        return "";

    return path;
}

LispLocalFile::LispLocalFile(
    LispEnvironment& environment,
    const std::string& fname,
    bool read,
    const std::vector<std::string>& dirs):
    environment(environment)
{
    std::string othername;

    if (read) {
        othername = fname;
        MapPathSeparators(othername);

        stream.open(othername, std::ios_base::in | std::ios_base::binary);

        for (std::size_t i = 0; !stream.is_open() && i < dirs.size(); ++i) {
            othername = dirs[i];
            othername += fname;
            MapPathSeparators(othername);
            stream.open(othername, std::ios_base::in | std::ios_base::binary);
        }
    } else {
        othername = fname;
        MapPathSeparators(othername);
        stream.open(othername, std::ios_base::out);
    }
}

LispLocalFile::~LispLocalFile()
{
    if (stream.is_open())
        stream.close();
}
<commit_msg>try to fix https://travis-ci.org/grzegorzmazur/yacas/jobs/345256414<commit_after>#include "yacas/platfileio.h"

#ifdef _WIN32
#define MAP_TO_WIN32_PATH_SEPARATOR
#endif // WIN32

static void MapPathSeparators(std::string& filename)
{
#ifdef MAP_TO_WIN32_PATH_SEPARATOR
    for (std::size_t i = 0; i < filename.size(); ++i)
        if (filename[i] == '/')
            filename[i] = '\\';
#endif
}


StdFileInput::StdFileInput(std::istream& stream, InputStatus& aStatus):
    LispInput(aStatus),
    _stream(stream)
{
}

StdFileInput::StdFileInput(LispLocalFile& file, InputStatus& aStatus):
    LispInput(aStatus),
    _stream(file.stream),
    _position(0),
    _cp_ready(false)
{
}

char32_t StdFileInput::Next()
{
    if (!_cp_ready)
        _get();

    if (EndOfStream())
        return std::char_traits<char32_t>::eof();
    
    _cp_ready = false;
    _position += 1;
    
    return _cp;
}

char32_t StdFileInput::Peek()
{
    if (EndOfStream())
        return std::char_traits<char32_t>::eof();

    if (!_cp_ready)
        _get();
    
    return _cp;
}

void StdFileInput::Rewind()
{
    _stream.seekg(0);
    _position = 0;
    _cp_ready = false;
}

bool StdFileInput::EndOfStream() const
{
    if (_stream.eof())
        return true;
    
    if (!_cp_ready)
        _get();
            
    return _stream.eof();
}

std::size_t StdFileInput::Position() const
{
    return _position;
}

void StdFileInput::SetPosition(std::size_t n)
{
    Rewind();
    
    for (std::size_t i = 0; i < n; ++i)
        Next();
}

void StdFileInput::_get() const
{
    char p[4];
    char* q = p;
    
    *q++ = _stream.get();
    
    while (!_stream.eof() && !utf8::is_valid(p, q))
        *q++ = _stream.get();
    
    if (_stream.eof())
        return;
    
    utf8::utf8to32(p, q, &_cp);
    _cp_ready = true;
}


std::string InternalFindFile(const std::string& fname, const std::vector<std::string>& dirs)
{
    std::string path(fname);

    MapPathSeparators(path);

    std::ifstream f(path);
    for (std::size_t i = 0; !f.good() && i < dirs.size(); ++i) {
        path = dirs[i] + fname;
        MapPathSeparators(path);
        f = std::move(std::ifstream(path));
    }

    if (!f.good())
        return "";

    return path;
}

LispLocalFile::LispLocalFile(
    LispEnvironment& environment,
    const std::string& fname,
    bool read,
    const std::vector<std::string>& dirs):
    environment(environment)
{
    std::string othername;

    if (read) {
        othername = fname;
        MapPathSeparators(othername);

        stream.open(othername, std::ios_base::in | std::ios_base::binary);

        for (std::size_t i = 0; !stream.is_open() && i < dirs.size(); ++i) {
            othername = dirs[i];
            othername += fname;
            MapPathSeparators(othername);
            stream.open(othername, std::ios_base::in | std::ios_base::binary);
        }
    } else {
        othername = fname;
        MapPathSeparators(othername);
        stream.open(othername, std::ios_base::out);
    }
}

LispLocalFile::~LispLocalFile()
{
    if (stream.is_open())
        stream.close();
}

{'fraction_non_alphanumeric': 0.15703824247355574, 'fraction_numerical': 0.011228641171684296, 'mean_word_length': 2.7339003645200486, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2210512', 'n_tokens_mistral': 2330, 'n_tokens_neox': 2253, 'n_words': 411}