<commit_before>/*
 * SourceCode.cpp
 * 
 * This file is part of the XShaderCompiler project (Copyright (c) 2014-2016 by <NAME>)
 * See "LICENSE.txt" for license information.
 */

#include "SourceCode.h"
#include <algorithm>


namespace Xsc
{


SourceCode::SourceCode(const std::shared_ptr<std::istream>& stream) :
    stream_{ stream }
{
}

bool SourceCode::IsValid() const
{
    return (stream_ != nullptr && stream_->good());
}

char SourceCode::Next()
{
    if (!IsValid())
        return 0;

    /* Check if reader is at end-of-line */
    while (pos_.Column() >= currentLine_.size())
    {
        /* Read new line in source file */
        std::getline(*stream_, currentLine_);
        currentLine_ += '\n';
        pos_.IncRow();

        /* Store current line for later reports */
        lines_.push_back(currentLine_);

        /* Check if end-of-file is reached */
        if (stream_->eof())
            return 0;
    }

    /* Increment column and return current character */
    auto chr = currentLine_[pos_.Column()];
    pos_.IncColumn();

    return chr;
}

static bool FinalizeMarker(
    const SourceArea& area, const std::string& lineIn, std::string& lineOut, std::string& markerOut)
{
    if (area.Pos().Column() >= lineIn.size() || area.Pos().Column() == 0 || area.Length() == 0)
        return false;

    /* Copy input line into output line */
    lineOut = lineIn;

    /* Replace all tabs with blanks after source area to make the line marker always fit ('\t' -> ' ') */
    for (auto i = static_cast<std::size_t>(area.Pos().Column()); i < lineOut.size(); ++i)<commit_msg>Enhanced "Next" function in SourceCode class to also retrieve the last line in a source file.<commit_after>/*
 * SourceCode.cpp
 * 
 * This file is part of the XShaderCompiler project (Copyright (c) 2014-2016 by <NAME>anns)
 * See "LICENSE.txt" for license information.
 */

#include "SourceCode.h"
#include <algorithm>


namespace Xsc
{


SourceCode::SourceCode(const std::shared_ptr<std::istream>& stream) :
    stream_{ stream }
{
}

bool SourceCode::IsValid() const
{
    return (stream_ != nullptr && stream_->good());
}

char SourceCode::Next()
{
    /* Check if reader is at end-of-line */
    while (pos_.Column() >= currentLine_.size())
    {
        /* Check if end-of-file is reached */
        if (!IsValid() || stream_->eof())
            return 0;

        /* Read new line in source file */
        std::getline(*stream_, currentLine_);
        currentLine_ += '\n';
        pos_.IncRow();

        /* Store current line for later reports */
        lines_.push_back(currentLine_);
    }

    /* Increment column and return current character */
    auto chr = currentLine_[pos_.Column()];
    pos_.IncColumn();

    return chr;
}

static bool FinalizeMarker(
    const SourceArea& area, const std::string& lineIn, std::string& lineOut, std::string& markerOut)
{
    if (area.Pos().Column() >= lineIn.size() || area.Pos().Column() == 0 || area.Length() == 0)
        return false;

    /* Copy input line into output line */
    lineOut = lineIn;

    /* Replace all tabs with blanks after source area to make the line marker always fit ('\t' -> ' ') */
    for (auto i = static_cast<std::size_t>(area.Pos().Column()); i < lineOut.size(); ++i)
{'fraction_non_alphanumeric': 0.14869431643625192, 'fraction_numerical': 0.00706605222734255, 'mean_word_length': 2.8947368421052633, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7432059', 'n_tokens_mistral': 1012, 'n_tokens_neox': 962, 'n_words': 301}