<commit_before>//-------------------------------------------------------------------------------------------------------
// Copyright (C) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
//-------------------------------------------------------------------------------------------------------
#pragma once

namespace Js
{
    struct Utf8SourceInfo : public FinalizableObject
    {
        typedef JsUtil::LeafValueDictionary<Js::LocalFunctionId, Js::FunctionBody*>::Type FunctionBodyDictionary;
        typedef JsUtil::LeafValueDictionary<Js::LocalFunctionId, Js::ParseableFunctionInfo*>::Type DeferredFunctionsDictionary;
        typedef JsUtil::List<Js::FunctionInfo *, Recycler> FunctionInfoList;

        friend class RemoteUtf8SourceInfo;
        friend class ScriptContext;
    public:
        bool HasSource() const { return !this->sourceHolder->IsEmpty(); }

        LPCUTF8 GetSource(const char16 * reason = nullptr) const;
        size_t GetCbLength(const char16 * reason = nullptr) const;

        ULONG GetParseFlags()
        {
            return this->parseFlags;
        }

        void SetParseFlags(ULONG parseFlags)
        {
            this->parseFlags = parseFlags;
        }

        ULONG GetByteCodeGenerationFlags()
        {
            return this->byteCodeGenerationFlags;
        }

        void SetByteCodeGenerationFlags(ULONG byteCodeGenerationFlags)
        {
            this->byteCodeGenerationFlags = byteCodeGenerationFlags;
        }

        bool IsInDebugMode() const
        {
            return (this->debugModeSource != nullptr || this->debugModeSourceIsEmpty) && this->m_isInDebugMode;
        }

        void SetInDebugMode(bool inDebugMode)
        {
            AssertMsg(!GetIsLibraryCode(), "Shouldn't call SetInDebugMode for Library code.");

            AssertMsg(this->sourceHolder != nullptr, "We have no source holder.");

            AssertMsg(this->m_isInDebugMode != inDebugMode, "Why are we setting same value");

            this->m_isInDebugMode = inDebugMode;

            if (!this->sourceHolder->IsDeferrable())
            {
                return;
            }

            if (inDebugMode)
            {
                this->debugModeSource = this->sourceHolder->GetSource(_u("Entering Debug Mode"));
                this->debugModeSourceLength = this->sourceHolder->GetByteLength(_u("Entering Debug Mode"));
                this->debugModeSourceIsEmpty = !this->HasSource() || this->debugModeSource == nullptr;
                this->EnsureLineOffsetCache();
            }
            else
            {
                // If debugModeSourceIsEmpty is false, that means debugModeSource isn't nullptr or we aren't in debug mode.
                this->debugModeSourceIsEmpty = false;
                this->debugModeSource = nullptr;
                this->debugModeSourceLength = 0;
            }
        }

        void RetrieveSourceText(__out_ecount_full(cchLim - cchMin) LPOLESTR cpText, charcount_t cchMin, charcount_t cchLim) const
        {
            size_t cbLength = GetCbLength(_u("Utf8SourceInfo::RetrieveSourceText"));
            LPCUTF8 source = GetSource(_u("Utf8SourceInfo::RetrieveSourceText"));
            LPCUTF8 pbStart = nullptr;
            LPCUTF8 pbEnd = nullptr;
            
            if (cbLength == GetCchLength())
            {
                pbStart = source + cchMin;
                pbEnd = source + cchLim;
            }
            else
            {
                pbStart = source + utf8::CharacterIndexToByteIndex(source, cbLength, cchMin, utf8::doAllowThreeByteSurrogates);
                pbEnd = source + utf8::CharacterIndexToByteIndex(source, cbLength, cchLim, utf8::doAllowThreeByteSurrogates);
            }
            
            utf8::DecodeUnitsInto(cpText, pbStart, pbEnd, utf8::doAllowThreeByteSurrogates);
        }

        size_t CharacterIndexToByteIndex(charcount_t cchIndex) const
        {
            return cchIndex < m_cchLength ? (GetCbLength(_u("CharacterIndexToByteIndex")) == m_cchLength ?  cchIndex : utf8::CharacterIndexToByteIndex(this->GetSource(_u("CharacterIndexToByteIndex")), GetCbLength(_u("CharacterIndexToByteIndex")), cchIndex, utf8::doAllowThreeByteSurrogates)) : m_cchLength;
        }

        charcount_t ByteIndexToCharacterIndex(size_t cbIndex) const
        {
            return cbIndex < GetCbLength(_u("CharacterIndexToByteIndex")) ? static_cast< charcount_t>(GetCbLength(_u("CharacterIndexToByteIndex")) == m_cchLength ? cbIndex : utf8::ByteIndexIntoCharacterIndex(this->GetSource(_u("CharacterIndexToByteIndex")), cbIndex, utf8::doAllowThreeByteSurrogates)) : static_cast< charcount_t >(GetCbLength(_u("CharacterIndexToByteIndex")));
        }

        charcount_t GetCchLength() const
        {
            return m_cchLength;
        }

        void SetCchLength(charcount_t cchLength)
        {
            m_cchLength = cchLength;
        }

        const SRCINFO* GetSrcInfo() const
        {
            return m_srcInfo;
        }

        void EnsureInitialized(int initialFunctionCount);
<commit_msg>[1.6>master] [MERGE #3294 @boingoing] Remove Utf8SourceInfo::RetrieveSourceText since it is dead code<commit_after>//-------------------------------------------------------------------------------------------------------
// Copyright (C) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
//-------------------------------------------------------------------------------------------------------
#pragma once

namespace Js
{
    struct Utf8SourceInfo : public FinalizableObject
    {
        typedef JsUtil::LeafValueDictionary<Js::LocalFunctionId, Js::FunctionBody*>::Type FunctionBodyDictionary;
        typedef JsUtil::LeafValueDictionary<Js::LocalFunctionId, Js::ParseableFunctionInfo*>::Type DeferredFunctionsDictionary;
        typedef JsUtil::List<Js::FunctionInfo *, Recycler> FunctionInfoList;

        friend class RemoteUtf8SourceInfo;
        friend class ScriptContext;
    public:
        bool HasSource() const { return !this->sourceHolder->IsEmpty(); }

        LPCUTF8 GetSource(const char16 * reason = nullptr) const;
        size_t GetCbLength(const char16 * reason = nullptr) const;

        ULONG GetParseFlags()
        {
            return this->parseFlags;
        }

        void SetParseFlags(ULONG parseFlags)
        {
            this->parseFlags = parseFlags;
        }

        ULONG GetByteCodeGenerationFlags()
        {
            return this->byteCodeGenerationFlags;
        }

        void SetByteCodeGenerationFlags(ULONG byteCodeGenerationFlags)
        {
            this->byteCodeGenerationFlags = byteCodeGenerationFlags;
        }

        bool IsInDebugMode() const
        {
            return (this->debugModeSource != nullptr || this->debugModeSourceIsEmpty) && this->m_isInDebugMode;
        }

        void SetInDebugMode(bool inDebugMode)
        {
            AssertMsg(!GetIsLibraryCode(), "Shouldn't call SetInDebugMode for Library code.");

            AssertMsg(this->sourceHolder != nullptr, "We have no source holder.");

            AssertMsg(this->m_isInDebugMode != inDebugMode, "Why are we setting same value");

            this->m_isInDebugMode = inDebugMode;

            if (!this->sourceHolder->IsDeferrable())
            {
                return;
            }

            if (inDebugMode)
            {
                this->debugModeSource = this->sourceHolder->GetSource(_u("Entering Debug Mode"));
                this->debugModeSourceLength = this->sourceHolder->GetByteLength(_u("Entering Debug Mode"));
                this->debugModeSourceIsEmpty = !this->HasSource() || this->debugModeSource == nullptr;
                this->EnsureLineOffsetCache();
            }
            else
            {
                // If debugModeSourceIsEmpty is false, that means debugModeSource isn't nullptr or we aren't in debug mode.
                this->debugModeSourceIsEmpty = false;
                this->debugModeSource = nullptr;
                this->debugModeSourceLength = 0;
            }
        }

        size_t CharacterIndexToByteIndex(charcount_t cchIndex) const
        {
            return cchIndex < m_cchLength ? (GetCbLength(_u("CharacterIndexToByteIndex")) == m_cchLength ?  cchIndex : utf8::CharacterIndexToByteIndex(this->GetSource(_u("CharacterIndexToByteIndex")), GetCbLength(_u("CharacterIndexToByteIndex")), cchIndex, utf8::doAllowThreeByteSurrogates)) : m_cchLength;
        }

        charcount_t ByteIndexToCharacterIndex(size_t cbIndex) const
        {
            return cbIndex < GetCbLength(_u("CharacterIndexToByteIndex")) ? static_cast< charcount_t>(GetCbLength(_u("CharacterIndexToByteIndex")) == m_cchLength ? cbIndex : utf8::ByteIndexIntoCharacterIndex(this->GetSource(_u("CharacterIndexToByteIndex")), cbIndex, utf8::doAllowThreeByteSurrogates)) : static_cast< charcount_t >(GetCbLength(_u("CharacterIndexToByteIndex")));
        }

        charcount_t GetCchLength() const
        {
            return m_cchLength;
        }

        void SetCchLength(charcount_t cchLength)
        {
            m_cchLength = cchLength;
        }

        const SRCINFO* GetSrcInfo() const
        {
            return m_srcInfo;
        }

        void EnsureInitialized(int initialFunctionCount);

{'fraction_non_alphanumeric': 0.13236540907164052, 'fraction_numerical': 0.004451038575667656, 'mean_word_length': 2.171034946236559, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5375001', 'n_tokens_mistral': 2491, 'n_tokens_neox': 2430, 'n_words': 480}