<commit_before>﻿// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Linq;
using Microsoft.Diagnostics.Runtime.ICorDebug;
using System.Threading;

#pragma warning disable 649

namespace Microsoft.Diagnostics.Runtime.Desktop
{
    internal enum DesktopVersion
    {
        v2,
        v4,
        v45
    }

    struct DomainContainer
    {
        public List<ClrAppDomain> Domains;
        public DesktopAppDomain System;
        public DesktopAppDomain Shared;
    }

    abstract internal class DesktopRuntimeBase : RuntimeBase
    {
        #region Variables
        protected CommonMethodTables _commonMTs;
        private Dictionary<uint, ICorDebug.ICorDebugThread> _corDebugThreads;
        private ClrModule[] _moduleList = null;
        private Lazy<List<ClrThread>> _threads;
        private Lazy<DesktopGCHeap> _heap;
        private Lazy<DesktopThreadPool> _threadpool;
        private ErrorModule _errorModule;
        private Lazy<DomainContainer> _appDomains;
        private Lazy<Dictionary<ulong, uint>> _moduleSizes;
        private Dictionary<ulong, DesktopModule> _modules = new Dictionary<ulong, DesktopModule>();
        private Dictionary<string, DesktopModule> _moduleFiles = new Dictionary<string, DesktopModule>();
        private Lazy<ClrModule> _mscorlib;
        #endregion

        internal DesktopRuntimeBase(ClrInfo info, DataTargetImpl dt, DacLibrary lib)
            : base(info, dt, lib)
        {
            _heap = new Lazy<DesktopGCHeap>(CreateHeap);
            _threads = new Lazy<List<ClrThread>>(CreateThreadList);
            _appDomains = new Lazy<DomainContainer>(CreateAppDomainList);
            _threadpool = new Lazy<DesktopThreadPool>(CreateThreadPoolData);
            _moduleSizes = new Lazy<Dictionary<ulong, uint>>(() => _dataReader.EnumerateModules().ToDictionary(module => module.ImageBase, module => module.FileSize));
            _mscorlib = new Lazy<ClrModule>(GetMscorlib);
        }


        /// <summary>
        /// Flushes the dac cache.  This function MUST be called any time you expect to call the same function
        /// but expect different results.  For example, after walking the heap, you need to call Flush before
        /// attempting to walk the heap again.
        /// </summary>
        public override void Flush()
        {
            OnRuntimeFlushed();

            Revision++;
            _dacInterface.Flush();
            
            MemoryReader = null;
            _moduleList = null;
            _threads = new Lazy<List<ClrThread>>(CreateThreadList);
            _appDomains = new Lazy<DomainContainer>(CreateAppDomainList);
            _heap = new Lazy<DesktopGCHeap>(CreateHeap);
            _threadpool = new Lazy<DesktopThreadPool>(CreateThreadPoolData);
            _mscorlib = new Lazy<ClrModule>(GetMscorlib);
        }

        internal ulong GetModuleSize(ulong address)
        {
            _moduleSizes.Value.TryGetValue(address, out uint size);
            return size;
        }

        internal int Revision { get; set; }

        public ErrorModule ErrorModule
        {
            get
            {
                if (_errorModule == null)
                    _errorModule = new ErrorModule(this);

                return _errorModule;
            }
        }

        internal override IGCInfo GetGCInfo()
        {
            IGCInfo data = GetGCInfoImpl();
            if (data == null)
            {
                throw new ClrDiagnosticsException("This runtime is not initialized and contains no data.", ClrDiagnosticsException.HR.RuntimeUninitialized);
            }

            return data;
        }

        public override IEnumerable<ClrException> EnumerateSerializedExceptions()
        {
            return new ClrException[0];
        }

        public override IEnumerable<int> EnumerateGCThreads()
        {
            foreach (uint thread in _dataReader.EnumerateAllThreads())
            {
                ulong teb = _dataReader.GetThreadTeb(thread);
                int threadType = DesktopThread.GetTlsSlotForThread(this, teb);
                if ((threadType & (int)DesktopThread.TlsThreadType.ThreadType_GC) == (int)DesktopThread.TlsThreadType.ThreadType_GC)
                    yield return (int)thread;
            }
        }


        /// <summary>
        /// Returns the version of the target process (v2, v4, v45)
        /// </summary>
        internal abstract DesktopVersion CLRVersion { get; }

        internal abstract IGCInfo GetGCInfoImpl();

        /// <summary>
        /// Returns the pointer size of the target process.
        /// </summary>
        public override int PointerSize
        {
            get
            {
                return IntPtr.Size;
            }
        }

        /// <summary>
        /// Returns the MethodTable for an array of objects.
        /// </summary>
        public ulong ArrayMethodTable
        {
            get
            {
                return _commonMTs.ArrayMethodTable;
            }
        }


        public override CcwData GetCcwDataByAddress(ulong addr)
        {
            var ccw = GetCCWData(addr);
            if (ccw == null)
                return null;

            return new DesktopCCWData((DesktopGCHeap)_heap.Value, addr, ccw);
        }

        internal ICorDebugThread GetCorDebugThread(uint osid)
        {
            if (_corDebugThreads == null)
            {
                _corDebugThreads = new Dictionary<uint, ICorDebugThread>();

                ICorDebugProcess process = CorDebugProcess;
                if (process == null)
                    return null;

                process.EnumerateThreads(out ICorDebugThreadEnum threadEnum);

                ICorDebugThread[] threads = new ICorDebugThread[1];
                while (threadEnum.Next(1, threads, out uint fetched) == 0 && fetched == 1)
                {
                    try
                    {
                        threads[0].GetID(out uint id);
                        _corDebugThreads[id] = threads[0];
                    }
                    catch
                    {
                    }
                }
            }

            _corDebugThreads.TryGetValue(osid, out ICorDebugThread result);
            return result;
        }

        public override IList<ClrAppDomain> AppDomains => _appDomains.Value.Domains;
        public override IList<ClrThread> Threads => _threads.Value;

        private List<ClrThread> CreateThreadList()
        {
            IThreadStoreData threadStore = GetThreadStoreData();
            ulong finalizer = ulong.MaxValue - 1;
            if (threadStore != null)
                finalizer = threadStore.Finalizer;

            List<ClrThread> threads = new List<ClrThread>();

            ulong addr = GetFirstThread();
            IThreadData thread = GetThread(addr);

            // Ensure we don't hit an infinite loop
            HashSet<ulong> seen = new HashSet<ulong> { addr };
            while (thread != null)
            {
                threads.Add(new DesktopThread(this, thread, addr, addr == finalizer));
                addr = thread.Next;
                if (seen.Contains(addr))
                    break;

                seen.Add(addr);
                thread = GetThread(addr);
            }

            return threads;
        }

        public ulong ExceptionMethodTable { get { return _commonMTs.ExceptionMethodTable; } }
        public ulong ObjectMethodTable
        {
            get
            {
                return _commonMTs.ObjectMethodTable;
            }
        }

        /// <summary>
        /// Returns the MethodTable for string objects.
        /// </summary>
        public ulong StringMethodTable
        {
            get
            {
                return _commonMTs.StringMethodTable;
            }
        }

        /// <summary>
        /// Returns the MethodTable for free space markers.
        /// </summary>
        public ulong FreeMethodTable
        {
            get
            {
                return _commonMTs.FreeMethodTable;
            }
        }

        public override ClrHeap Heap => _heap.Value;

        [Obsolete]
        public override ClrHeap GetHeap() => _heap.Value;

        private DesktopGCHeap CreateHeap()
        {
            if (HasArrayComponentMethodTables)
                return new LegacyGCHeap(this);
            else
                return  new V46GCHeap(this);
        }

        public override ClrThreadPool ThreadPool => _threadpool.Value;

        [Obsolete]
        public override ClrThreadPool GetThreadPool() => _threadpool.Value;
        
        public ulong SystemDomainAddress => _appDomains.Value.System.Address;
        public ulong SharedDomainAddress => _appDomains.Value.Shared.Address;
        public override ClrAppDomain SystemDomain => _appDomains.Value.System;
        public override ClrAppDomain SharedDomain => _appDomains.Value.Shared;
        public bool IsSingleDomain => _appDomains.Value.Domains.Count == 1;

        public override ClrMethod GetMethodByHandle(ulong methodHandle)
        {
            if (methodHandle == 0)
                return null;

            IMethodDescData methodDesc = GetMethodDescData(methodHandle);
            if (methodDesc == null)
                return null;

            ClrType type = Heap.GetTypeByMethodTable(methodDesc.MethodTable);
            if (type == null)
                return null;

            return type.GetMethod(methodDesc.MDToken);
        }

        /// <summary>
        /// Enumerates regions of memory which CLR has allocated with a description of what data
        /// resides at that location.  Note that this does not return every chunk of address space
        /// that CLR allocates.
        /// </summary>
        /// <returns>An enumeration of memory regions in the process.</returns>
        public override IEnumerable<ClrMemoryRegion> EnumerateMemoryRegions()
        {
            // Enumerate GC Segment regions.
            IHeapDetails[] heaps;
            if (ServerGC)
            {
                heaps = new IHeapDetails[HeapCount];
                int i = 0;
                ulong[] heapList = GetServerHeapList();
                if (heapList != null)
                {
                    foreach (ulong addr in heapList)
                    {
                        heaps[i++] = GetSvrHeapDetails(addr);
                        if (i == heaps.Length)
                            break;
                    }
                }
                else
                {
                    heaps = new IHeapDetails[0];
                }
            }
            else
            {
                Debug.Assert(HeapCount == 1);
                heaps = new IHeapDetails[1];
                heaps[0] = GetWksHeapDetails();
            }

            int max = 2048;  // Max number of segments in case of inconsistent data.
            for (int i = 0; i < heaps.Length; ++i)
            {
                // Small heap
                ISegmentData segment = GetSegmentData(heaps[i].FirstHeapSegment);
                while (segment != null && max-- > 0)
                {
                    Debug.Assert(segment.Start < segment.Committed);
                    Debug.Assert(segment.Committed <= segment.Reserved);

                    GCSegmentType type = (segment.Address == heaps[i].EphemeralSegment) ? GCSegmentType.Ephemeral : GCSegmentType.Regular;
                    yield return new MemoryRegion(this, segment.Start, segment.Committed - segment.Start, ClrMemoryRegionType.GCSegment, (uint)i, type);
                    yield return new MemoryRegion(this, segment.Committed, segment.Reserved - segment.Committed, ClrMemoryRegionType.ReservedGCSegment, (uint)i, type);

                    if (segment.Address == segment.Next || segment.Address == 0)
                        segment = null;
                    else
                        segment = GetSegmentData(segment.Next);
                }

                segment = GetSegmentData(heaps[i].FirstLargeHeapSegment);
                while (segment != null && max-- > 0)
                {
                    Debug.Assert(segment.Start < segment.Committed);
                    Debug.Assert(segment.Committed <= segment.Reserved);

                    yield return new MemoryRegion(this, segment.Start, segment.Committed - segment.Start, ClrMemoryRegionType.GCSegment, (uint)i, GCSegmentType.LargeObject);
                    yield return new MemoryRegion(this, segment.Committed, segment.Reserved - segment.Committed, ClrMemoryRegionType.ReservedGCSegment, (uint)i, GCSegmentType.LargeObject);

                    if (segment.Address == segment.Next || segment.Address == 0)
                        segment = null;
                    else
                        segment = GetSegmentData(segment.Next);
                }
            }

            // Enumerate handle table regions.
            HashSet<ulong> regions = new HashSet<ulong>();
            foreach (ClrHandle handle in EnumerateHandles())
            {
                if (!_dataReader.VirtualQuery(handle.Address, out VirtualQueryData vq))
                    continue;
<commit_msg>Fix assert enumerating memory regions<commit_after>﻿// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Linq;
using Microsoft.Diagnostics.Runtime.ICorDebug;
using System.Threading;

#pragma warning disable 649

namespace Microsoft.Diagnostics.Runtime.Desktop
{
    internal enum DesktopVersion
    {
        v2,
        v4,
        v45
    }

    struct DomainContainer
    {
        public List<ClrAppDomain> Domains;
        public DesktopAppDomain System;
        public DesktopAppDomain Shared;
    }

    abstract internal class DesktopRuntimeBase : RuntimeBase
    {
        #region Variables
        protected CommonMethodTables _commonMTs;
        private Dictionary<uint, ICorDebug.ICorDebugThread> _corDebugThreads;
        private ClrModule[] _moduleList = null;
        private Lazy<List<ClrThread>> _threads;
        private Lazy<DesktopGCHeap> _heap;
        private Lazy<DesktopThreadPool> _threadpool;
        private ErrorModule _errorModule;
        private Lazy<DomainContainer> _appDomains;
        private Lazy<Dictionary<ulong, uint>> _moduleSizes;
        private Dictionary<ulong, DesktopModule> _modules = new Dictionary<ulong, DesktopModule>();
        private Dictionary<string, DesktopModule> _moduleFiles = new Dictionary<string, DesktopModule>();
        private Lazy<ClrModule> _mscorlib;
        #endregion

        internal DesktopRuntimeBase(ClrInfo info, DataTargetImpl dt, DacLibrary lib)
            : base(info, dt, lib)
        {
            _heap = new Lazy<DesktopGCHeap>(CreateHeap);
            _threads = new Lazy<List<ClrThread>>(CreateThreadList);
            _appDomains = new Lazy<DomainContainer>(CreateAppDomainList);
            _threadpool = new Lazy<DesktopThreadPool>(CreateThreadPoolData);
            _moduleSizes = new Lazy<Dictionary<ulong, uint>>(() => _dataReader.EnumerateModules().ToDictionary(module => module.ImageBase, module => module.FileSize));
            _mscorlib = new Lazy<ClrModule>(GetMscorlib);
        }


        /// <summary>
        /// Flushes the dac cache.  This function MUST be called any time you expect to call the same function
        /// but expect different results.  For example, after walking the heap, you need to call Flush before
        /// attempting to walk the heap again.
        /// </summary>
        public override void Flush()
        {
            OnRuntimeFlushed();

            Revision++;
            _dacInterface.Flush();
            
            MemoryReader = null;
            _moduleList = null;
            _threads = new Lazy<List<ClrThread>>(CreateThreadList);
            _appDomains = new Lazy<DomainContainer>(CreateAppDomainList);
            _heap = new Lazy<DesktopGCHeap>(CreateHeap);
            _threadpool = new Lazy<DesktopThreadPool>(CreateThreadPoolData);
            _mscorlib = new Lazy<ClrModule>(GetMscorlib);
        }

        internal ulong GetModuleSize(ulong address)
        {
            _moduleSizes.Value.TryGetValue(address, out uint size);
            return size;
        }

        internal int Revision { get; set; }

        public ErrorModule ErrorModule
        {
            get
            {
                if (_errorModule == null)
                    _errorModule = new ErrorModule(this);

                return _errorModule;
            }
        }

        internal override IGCInfo GetGCInfo()
        {
            IGCInfo data = GetGCInfoImpl();
            if (data == null)
            {
                throw new ClrDiagnosticsException("This runtime is not initialized and contains no data.", ClrDiagnosticsException.HR.RuntimeUninitialized);
            }

            return data;
        }

        public override IEnumerable<ClrException> EnumerateSerializedExceptions()
        {
            return new ClrException[0];
        }

        public override IEnumerable<int> EnumerateGCThreads()
        {
            foreach (uint thread in _dataReader.EnumerateAllThreads())
            {
                ulong teb = _dataReader.GetThreadTeb(thread);
                int threadType = DesktopThread.GetTlsSlotForThread(this, teb);
                if ((threadType & (int)DesktopThread.TlsThreadType.ThreadType_GC) == (int)DesktopThread.TlsThreadType.ThreadType_GC)
                    yield return (int)thread;
            }
        }


        /// <summary>
        /// Returns the version of the target process (v2, v4, v45)
        /// </summary>
        internal abstract DesktopVersion CLRVersion { get; }

        internal abstract IGCInfo GetGCInfoImpl();

        /// <summary>
        /// Returns the pointer size of the target process.
        /// </summary>
        public override int PointerSize
        {
            get
            {
                return IntPtr.Size;
            }
        }

        /// <summary>
        /// Returns the MethodTable for an array of objects.
        /// </summary>
        public ulong ArrayMethodTable
        {
            get
            {
                return _commonMTs.ArrayMethodTable;
            }
        }


        public override CcwData GetCcwDataByAddress(ulong addr)
        {
            var ccw = GetCCWData(addr);
            if (ccw == null)
                return null;

            return new DesktopCCWData((DesktopGCHeap)_heap.Value, addr, ccw);
        }

        internal ICorDebugThread GetCorDebugThread(uint osid)
        {
            if (_corDebugThreads == null)
            {
                _corDebugThreads = new Dictionary<uint, ICorDebugThread>();

                ICorDebugProcess process = CorDebugProcess;
                if (process == null)
                    return null;

                process.EnumerateThreads(out ICorDebugThreadEnum threadEnum);

                ICorDebugThread[] threads = new ICorDebugThread[1];
                while (threadEnum.Next(1, threads, out uint fetched) == 0 && fetched == 1)
                {
                    try
                    {
                        threads[0].GetID(out uint id);
                        _corDebugThreads[id] = threads[0];
                    }
                    catch
                    {
                    }
                }
            }

            _corDebugThreads.TryGetValue(osid, out ICorDebugThread result);
            return result;
        }

        public override IList<ClrAppDomain> AppDomains => _appDomains.Value.Domains;
        public override IList<ClrThread> Threads => _threads.Value;

        private List<ClrThread> CreateThreadList()
        {
            IThreadStoreData threadStore = GetThreadStoreData();
            ulong finalizer = ulong.MaxValue - 1;
            if (threadStore != null)
                finalizer = threadStore.Finalizer;

            List<ClrThread> threads = new List<ClrThread>();

            ulong addr = GetFirstThread();
            IThreadData thread = GetThread(addr);

            // Ensure we don't hit an infinite loop
            HashSet<ulong> seen = new HashSet<ulong> { addr };
            while (thread != null)
            {
                threads.Add(new DesktopThread(this, thread, addr, addr == finalizer));
                addr = thread.Next;
                if (seen.Contains(addr))
                    break;

                seen.Add(addr);
                thread = GetThread(addr);
            }

            return threads;
        }

        public ulong ExceptionMethodTable { get { return _commonMTs.ExceptionMethodTable; } }
        public ulong ObjectMethodTable
        {
            get
            {
                return _commonMTs.ObjectMethodTable;
            }
        }

        /// <summary>
        /// Returns the MethodTable for string objects.
        /// </summary>
        public ulong StringMethodTable
        {
            get
            {
                return _commonMTs.StringMethodTable;
            }
        }

        /// <summary>
        /// Returns the MethodTable for free space markers.
        /// </summary>
        public ulong FreeMethodTable
        {
            get
            {
                return _commonMTs.FreeMethodTable;
            }
        }

        public override ClrHeap Heap => _heap.Value;

        [Obsolete]
        public override ClrHeap GetHeap() => _heap.Value;

        private DesktopGCHeap CreateHeap()
        {
            if (HasArrayComponentMethodTables)
                return new LegacyGCHeap(this);
            else
                return  new V46GCHeap(this);
        }

        public override ClrThreadPool ThreadPool => _threadpool.Value;

        [Obsolete]
        public override ClrThreadPool GetThreadPool() => _threadpool.Value;
        
        public ulong SystemDomainAddress => _appDomains.Value.System.Address;
        public ulong SharedDomainAddress => _appDomains.Value.Shared.Address;
        public override ClrAppDomain SystemDomain => _appDomains.Value.System;
        public override ClrAppDomain SharedDomain => _appDomains.Value.Shared;
        public bool IsSingleDomain => _appDomains.Value.Domains.Count == 1;

        public override ClrMethod GetMethodByHandle(ulong methodHandle)
        {
            if (methodHandle == 0)
                return null;

            IMethodDescData methodDesc = GetMethodDescData(methodHandle);
            if (methodDesc == null)
                return null;

            ClrType type = Heap.GetTypeByMethodTable(methodDesc.MethodTable);
            if (type == null)
                return null;

            return type.GetMethod(methodDesc.MDToken);
        }

        /// <summary>
        /// Enumerates regions of memory which CLR has allocated with a description of what data
        /// resides at that location.  Note that this does not return every chunk of address space
        /// that CLR allocates.
        /// </summary>
        /// <returns>An enumeration of memory regions in the process.</returns>
        public override IEnumerable<ClrMemoryRegion> EnumerateMemoryRegions()
        {
            // Enumerate GC Segment regions.
            IHeapDetails[] heaps;
            if (ServerGC)
            {
                heaps = new IHeapDetails[HeapCount];
                int i = 0;
                ulong[] heapList = GetServerHeapList();
                if (heapList != null)
                {
                    foreach (ulong addr in heapList)
                    {
                        heaps[i++] = GetSvrHeapDetails(addr);
                        if (i == heaps.Length)
                            break;
                    }
                }
                else
                {
                    heaps = new IHeapDetails[0];
                }
            }
            else
            {
                Debug.Assert(HeapCount == 1);
                heaps = new IHeapDetails[1];
                heaps[0] = GetWksHeapDetails();
            }

            HashSet<ulong> addresses = new HashSet<ulong>();
            for (int i = 0; i < heaps.Length; ++i)
            {
                // Small heap
                ISegmentData segment = GetSegmentData(heaps[i].FirstHeapSegment);
                while (segment != null)
                {
                    Debug.Assert(segment.Start < segment.Committed);

                    GCSegmentType type = (segment.Address == heaps[i].EphemeralSegment) ? GCSegmentType.Ephemeral : GCSegmentType.Regular;
                    yield return new MemoryRegion(this, segment.Start, segment.Committed - segment.Start, ClrMemoryRegionType.GCSegment, (uint)i, type);

                    if (segment.Committed <= segment.Reserved)
                        yield return new MemoryRegion(this, segment.Committed, segment.Reserved - segment.Committed, ClrMemoryRegionType.ReservedGCSegment, (uint)i, type);

                    if (segment.Address == segment.Next || segment.Address == 0)
                    {
                        break;
                    }
                    else
                    {
                        if (!addresses.Add(segment.Next))
                            break;

                        segment = GetSegmentData(segment.Next);
                    }
                }

                segment = GetSegmentData(heaps[i].FirstLargeHeapSegment);
                while (segment != null)
                {
                    Debug.Assert(segment.Start < segment.Committed);

                    yield return new MemoryRegion(this, segment.Start, segment.Committed - segment.Start, ClrMemoryRegionType.GCSegment, (uint)i, GCSegmentType.LargeObject);

                    if (segment.Committed <= segment.Reserved)
                        yield return new MemoryRegion(this, segment.Committed, segment.Reserved - segment.Committed, ClrMemoryRegionType.ReservedGCSegment, (uint)i, GCSegmentType.LargeObject);

                    if (segment.Address == segment.Next || segment.Address == 0)
                    {
                        break;
                    }
                    else
                    {
                        if (!addresses.Add(segment.Next))
                            break;

                        segment = GetSegmentData(segment.Next);
                    }
                }
            }

            // Enumerate handle table regions.
            HashSet<ulong> regions = new HashSet<ulong>();
            foreach (ClrHandle handle in EnumerateHandles())
            {
                if (!_dataReader.VirtualQuery(handle.Address, out VirtualQueryData vq))
                    continue;

{'fraction_non_alphanumeric': 0.08058459726108085, 'fraction_numerical': 0.0025033132086585185, 'mean_word_length': 1.6100115295926212, 'pattern_counts': {'":': 0, '<': 129, '<?xml version=': 0, '>': 149, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7484229', 'n_tokens_mistral': 7011, 'n_tokens_neox': 6636, 'n_words': 1656}