<commit_before>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2018.  All rights reserved
 */
#include "../../../StroikaPreComp.h"

#include "../../../Characters/Format.h"
#include "../../../Characters/ToString.h"
#include "../../../Execution/Common.h"
#include "../../../Execution/OperationNotSupportedException.h"
#include "../../../Memory/SmallStackBuffer.h"

#include "MessageStartTextInputStreamBinaryAdapter.h"

using namespace Stroika::Foundation;
using namespace Stroika::Foundation::Characters;
using namespace Stroika::Foundation::Execution;
using namespace Stroika::Foundation::IO;
using namespace Stroika::Foundation::IO::Network;
using namespace Stroika::Foundation::IO::Network::HTTP;
using namespace Stroika::Foundation::Streams;

// Comment this in to turn on aggressive noisy DbgTrace in this module
//#define USE_NOISY_TRACE_IN_THIS_MODULE_ 1

namespace {
    constexpr size_t kDefaultBufSize_ = 2 * 1024; // guess enough for http headers (typically around .8K but little cost in reserving a bit more)
}

//@todo - NOTE - NOT RIGHT - RE_READ RFP - maybe need to do mime decoding???
// http://stackoverflow.com/questions/4400678/http-header-should-use-what-character-encoding
// but for now this seems and adequate hack

class MessageStartTextInputStreamBinaryAdapter::Rep_ : public InputStream<Character>::_IRep, protected Debug::AssertExternallySynchronizedLock {
public:
    Rep_ (const InputStream<Byte>::Ptr& src)
        : fSource_ (src)
        , fAllDataReadBuf_ (kDefaultBufSize_)
        , fOffset_ (0)
        , fBufferFilledUpValidBytes_ (0)
    {
    }

public:
    bool AssureHeaderSectionAvailable ()
    {
#if USE_NOISY_TRACE_IN_THIS_MODULE_
        Debug::TraceContextBumper ctx (Stroika_Foundation_Debug_OptionalizeTraceArgs (L"MessageStartTextInputStreamBinaryAdapter::AssureHeaderSectionAvailable"));
#endif
        this->SeekRead (Whence::eFromStart, 0);
        bool      gotBareCRLFCRLF = false;
        Character c;
        enum state {
            gotCR,
            gotCRLF,
            gotCRLFCR,
            gotNOTHING,
        };
        state s = gotNOTHING;
        while (Memory::Optional<size_t> o = ReadNonBlocking (&c, &c + 1)) {
            if (*o == 0) {
                return true; // tricky corner case - EOF in header - treat as available so we process whole header
            }
            Assert (*o == 1);
            switch (c.GetCharacterCode ()) {
                case '\r': {
                    switch (s) {
                        case gotNOTHING: {
                            s = gotCR;
                        } break;
                        case gotCRLF: {
                            s = gotCRLFCR;
                        } break;
                        default: {
                            DbgTrace (L"Looks like bad HTTP header");
                            s = gotNOTHING;
                        } break;
                    }
                } break;
                case '\n': {
                    switch (s) {
                        case gotCR: {
                            s = gotCRLF;
                        } break;
                        case gotCRLFCR: {
                            gotBareCRLFCRLF = true;
                        } break;
                        default: {
                            DbgTrace (L"Looks like bad HTTP header");
                            s = gotNOTHING;
                        } break;
                    }
                } break;
                default: {
                    s = gotNOTHING;
                } break;
            }
        }
        this->SeekRead (Whence::eFromStart, 0);
        return gotBareCRLFCRLF;
    }

public:
    nonvirtual Characters::String ToString (ToStringFormat format) const
    {
        shared_lock<const AssertExternallySynchronizedLock> critSec{*this};
        StringBuilder                                       sb;
        sb += L"{";
        sb += L"Offset: " + Characters::Format (L"%d", fOffset_) + L", ";
        sb += L"HighWaterMark: " + Characters::Format (L"%d", fBufferFilledUpValidBytes_) + L", ";
        sb += L"TEXT: ";
        switch (format) {
            case ToStringFormat::eAsBytes: {
                for (size_t i = 0; i < fBufferFilledUpValidBytes_; ++i) {
                    sb += Characters::Format (L"x%x, ", fAllDataReadBuf_[i]);
                }
            } break;
            case ToStringFormat::eAsString: {
                sb += L"'";
                for (Character c : String::FromISOLatin1 (reinterpret_cast<const char*> (begin (fAllDataReadBuf_)), reinterpret_cast<const char*> (begin (fAllDataReadBuf_) + fBufferFilledUpValidBytes_))) {
                    switch (c.GetCharacterCode ()) {
                        case '\r':
                            sb += L"\\r";
                            break;
                        case '\n':
                            sb += L"\\n";
                            break;
                        default:
                            sb += c.GetCharacterCode ();<commit_msg>fixed bug with MessageStartTextInputStreamBinaryAdapter::AssureHeaderSectionAvailable which caused it to read too far (only noticible on POSTs)<commit_after>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2018.  All rights reserved
 */
#include "../../../StroikaPreComp.h"

#include "../../../Characters/Format.h"
#include "../../../Characters/ToString.h"
#include "../../../Execution/Common.h"
#include "../../../Execution/OperationNotSupportedException.h"
#include "../../../Memory/SmallStackBuffer.h"

#include "MessageStartTextInputStreamBinaryAdapter.h"

using namespace Stroika::Foundation;
using namespace Stroika::Foundation::Characters;
using namespace Stroika::Foundation::Execution;
using namespace Stroika::Foundation::IO;
using namespace Stroika::Foundation::IO::Network;
using namespace Stroika::Foundation::IO::Network::HTTP;
using namespace Stroika::Foundation::Streams;

// Comment this in to turn on aggressive noisy DbgTrace in this module
//#define USE_NOISY_TRACE_IN_THIS_MODULE_ 1

namespace {
    constexpr size_t kDefaultBufSize_ = 2 * 1024; // guess enough for http headers (typically around .8K but little cost in reserving a bit more)
}

//@todo - NOTE - NOT RIGHT - RE_READ RFP - maybe need to do mime decoding???
// http://stackoverflow.com/questions/4400678/http-header-should-use-what-character-encoding
// but for now this seems and adequate hack

class MessageStartTextInputStreamBinaryAdapter::Rep_ : public InputStream<Character>::_IRep, protected Debug::AssertExternallySynchronizedLock {
public:
    Rep_ (const InputStream<Byte>::Ptr& src)
        : fSource_ (src)
        , fAllDataReadBuf_ (kDefaultBufSize_)
        , fOffset_ (0)
        , fBufferFilledUpValidBytes_ (0)
    {
    }

public:
    bool AssureHeaderSectionAvailable ()
    {
#if USE_NOISY_TRACE_IN_THIS_MODULE_
        Debug::TraceContextBumper ctx (Stroika_Foundation_Debug_OptionalizeTraceArgs (L"MessageStartTextInputStreamBinaryAdapter::AssureHeaderSectionAvailable"));
#endif
        this->SeekRead (Whence::eFromStart, 0);
        Character c;
        enum state {
            gotCR,
            gotCRLF,
            gotCRLFCR,
            gotNOTHING,
        };
        state s = gotNOTHING;
        while (Memory::Optional<size_t> o = ReadNonBlocking (&c, &c + 1)) {
            if (*o == 0) {
                return true; // tricky corner case - EOF in header - treat as available so we process whole header
            }
            Assert (*o == 1);
            switch (c.GetCharacterCode ()) {
                case '\r': {
                    switch (s) {
                        case gotNOTHING: {
                            s = gotCR;
                        } break;
                        case gotCRLF: {
                            s = gotCRLFCR;
                        } break;
                        default: {
                            DbgTrace (L"Looks like bad HTTP header");
                            s = gotNOTHING;
                        } break;
                    }
                } break;
                case '\n': {
                    switch (s) {
                        case gotCR: {
                            s = gotCRLF;
                        } break;
                        case gotCRLFCR: {
                            this->SeekRead (Whence::eFromStart, 0);
                            return true;
                        } break;
                        default: {
                            DbgTrace (L"Looks like bad HTTP header");
                            s = gotNOTHING;
                        } break;
                    }
                } break;
                default: {
                    s = gotNOTHING;
                } break;
            }
        }
        return false;
    }

public:
    nonvirtual Characters::String ToString (ToStringFormat format) const
    {
        shared_lock<const AssertExternallySynchronizedLock> critSec{*this};
        StringBuilder                                       sb;
        sb += L"{";
        sb += L"Offset: " + Characters::Format (L"%d", fOffset_) + L", ";
        sb += L"HighWaterMark: " + Characters::Format (L"%d", fBufferFilledUpValidBytes_) + L", ";
        sb += L"TEXT: ";
        switch (format) {
            case ToStringFormat::eAsBytes: {
                for (size_t i = 0; i < fBufferFilledUpValidBytes_; ++i) {
                    sb += Characters::Format (L"x%x, ", fAllDataReadBuf_[i]);
                }
            } break;
            case ToStringFormat::eAsString: {
                sb += L"'";
                for (Character c : String::FromISOLatin1 (reinterpret_cast<const char*> (begin (fAllDataReadBuf_)), reinterpret_cast<const char*> (begin (fAllDataReadBuf_) + fBufferFilledUpValidBytes_))) {
                    switch (c.GetCharacterCode ()) {
                        case '\r':
                            sb += L"\\r";
                            break;
                        case '\n':
                            sb += L"\\n";
                            break;
                        default:
                            sb += c.GetCharacterCode ();
{'fraction_non_alphanumeric': 0.10715694006309148, 'fraction_numerical': 0.00611198738170347, 'mean_word_length': 1.5419694312202457, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3321423', 'n_tokens_mistral': 2773, 'n_tokens_neox': 2621, 'n_words': 639}