<commit_before>﻿namespace Dump2SQLite
{
    using System;
    using System.Collections.Generic;
    using System.Diagnostics;
    using System.Text;
    using Microsoft.Diagnostics.Runtime;

    internal static unsafe class Program
    {
        private static ulong currentObjectBeingTraversed = 0;

        private static sqlite3_stmt* insertObjectReferencesStmt;

        private static void Main(string[] args)
        {
            if (args.Length < 2)
            {
                Console.WriteLine("@ Dump2SQLite - Serialize GCHeap to SQLite DB @");
                Console.WriteLine(@"Usage: Dump2SQLite \path\to\filename.dmp \path\to\outputfile.db [DacDllLocation]");
                Console.WriteLine(@"Example 1: Dump2SQLite C:\w3wp.dmp C:\w3wp.db");
                Console.WriteLine(@"Example 2: Dump2SQLite C:\w3wp.dmp C:\w3wp.db C:\Windows\Microsoft.NET\Framework64\v4.0.30319\mscordacwks.dll");
                return;
            }

            string dumpLocation = args[0];

            Stopwatch watch = new Stopwatch();
            watch.Start();

            LogInfoWithTimeStamp("Loading Crash Dump from: " + dumpLocation + " ...");

            DataTarget target = DataTarget.LoadCrashDump(dumpLocation);
            var dacLocation = target.ClrVersions[0].LocalMatchingDac;

            if (args.Length == 3)
            {
                dacLocation = args[2];
            }

            if (!System.IO.File.Exists(dacLocation))
            {
                Console.WriteLine($"ERROR: Dac Location {dacLocation} does not exist, please point to DAC associated with the dump, you can also try your local DAC (%WINDIR%\\Microsoft.NET\\Framework64\\v4.0.30319\\mscordacwks.dll)");
            }

            var runtime = target.ClrVersions[0].CreateRuntime(dacLocation, ignoreMismatch: true);

            var heap = runtime.Heap;
            if (!heap.CanWalkHeap)
            {
                LogErrorWithTimeStamp("Heap is not walkable. Please collect a new dump.");
                return;
            }

            LogInfoWithTimeStamp("Crash Dump loaded.");

            var perTypeCounts = new Dictionary<ClrType, int>(1000000);

            var fileName = args[1];

            if (System.IO.File.Exists(fileName))
            {
                System.IO.File.Delete(fileName);
            }

            LogInfoWithTimeStamp("Creating SQLite Database filename: " + fileName + " ...");

            sqlite3* db;
            int error;
            if ((error = NativeMethods.sqlite3_open(fileName, out db)) != 0)
            {
                LogErrorWithTimeStamp("sqlite3_open failed with error code: " + error);
            }

            LogInfoWithTimeStamp("Successfully created SQLite Database filename: " + fileName);

            LogInfoWithTimeStamp("Creating SQLite Tables ...");

            if (!CreateTables(db))
            {
                return;
            }

            LogInfoWithTimeStamp("Successfully created SQLite Tables.");

            error = NativeMethods.sqlite3_exec(db, "BEGIN TRANSACTION;", IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
            if (error != 0)
            {
                LogErrorWithTimeStamp("sqlite3_exec -> BEGIN TRANSACTION; failed to execute with SQLite error code: " + error);
            }

            sqlite3_stmt* insertObjectsStmt, insertTypesStmt, insertRootsStmt, insertBlockingObjectsStmt, insertExceptionsStmt, insertThreadsStmt;

            if (!PrepareInsertStatement(db, out insertTypesStmt, @"INSERT INTO Types(TypeIndex, Count, Size, Name) VALUES (@1, @2, @3, @4);"))
            {
                return;
            }

            if (!PrepareInsertStatement(db, out insertObjectsStmt, @"INSERT INTO Objects(ObjectId, TypeIndex, Size) VALUES (@1, @2, @3);"))
            {
                return;
            }

            if (!PrepareInsertStatement(db, out insertObjectReferencesStmt, @"INSERT INTO ObjectReferences(ObjectId, ObjectReference) VALUES (@1, @2);"))
            {
                return;
            }

            if (!PrepareInsertStatement(db, out insertRootsStmt, @"INSERT INTO Roots(TypeIndex, ObjectId, Address, AppDomainId, ManagedThreadId, IsInterior, IsPinned, IsPossibleFalsePositive, GCRootKind, Name) VALUES (@1, @2, @3, @4, @5, @6, @7, @8, @9, @10);"))
            {
                return;
            }

            if (!PrepareInsertStatement(db, out insertBlockingObjectsStmt, @"INSERT INTO BlockingObjects(ObjectId, Taken, RecursionCount, Owner, HasSingleOwner, ThreadOwnerIds, ThreadWaiterIds, BlockingReason) VALUES (@1, @2, @3, @4, @5, @6, @7, @8);"))
            {
                return;
            }

            /*
            if (!PrepareInsertStatement(db, out insertExceptionsStmt, @"INSERT INTO Exceptions(ExceptionId, TypeId, Type, Message, Address, InnerExceptionId, HResult, StackId, StackTrace) VALUES (@1, @2, @3, @4, @5, @6, @7, @8, @9);"))
            {
                return;
            }

            if (!PrepareInsertStatement(db, out insertThreadsStmt, @"(GcMode, IsFinalizer, Address, IsAlive, OSThreadId, ManagedThreadId, AppDomain, LockCount, Teb, StackBase, StackLimit, StackId, ExceptionId, IsGC, IsDebuggerHelper, IsThreadpoolTimer, IsThreadpoolCompletionPort, IsThreadpoolWorker, IsThreadpoolWait, IsThreadpoolGate, IsSuspendingEE, IsShutdownHelper, IsAbortRequested, IsAborted, ISGCSuspendPending, IsDebugSuspended, IsBackground, IsUnstarted, IsCoInitialized, IsSTA, IsMTA, BlockingObjects, Roots) VALUES (@1, @2, @3, @4, @5, @6, @7, @8, @9, @10);"))
            {
                return;
            }
            */

            LogInfoWithTimeStamp("Starting to populate Objects Table ...");

            long objectCount = 0;
            var objRefWalk = new Action<ulong, int>(ObjRefWalk);
            foreach (var obj in heap.EnumerateObjects())
            {
                objectCount++;
                if (objectCount % 1000000 == 0)
                {
                    LogInfoWithTimeStamp($"Enumerated objects: {objectCount}");
                }

                var type = heap.GetObjectType(obj.Address);<commit_msg>comment cleanups<commit_after>﻿namespace Dump2SQLite
{
    using System;
    using System.Collections.Generic;
    using System.Diagnostics;
    using System.Text;
    using Microsoft.Diagnostics.Runtime;

    internal static unsafe class Program
    {
        private static ulong currentObjectBeingTraversed;

        private static sqlite3_stmt* insertObjectReferencesStmt;

        private static void Main(string[] args)
        {
            if (args.Length < 2)
            {
                Console.WriteLine("@ Dump2SQLite - Serialize GCHeap to SQLite DB @");
                Console.WriteLine(@"Usage: Dump2SQLite \path\to\filename.dmp \path\to\outputfile.db [DacDllLocation]");
                Console.WriteLine(@"Example 1: Dump2SQLite C:\w3wp.dmp C:\w3wp.db");
                Console.WriteLine(@"Example 2: Dump2SQLite C:\w3wp.dmp C:\w3wp.db C:\Windows\Microsoft.NET\Framework64\v4.0.30319\mscordacwks.dll");
                return;
            }

            string dumpLocation = args[0];

            Stopwatch watch = new Stopwatch();
            watch.Start();

            LogInfoWithTimeStamp("Loading Crash Dump from: " + dumpLocation + " ...");

            DataTarget target = DataTarget.LoadCrashDump(dumpLocation);
            var dacLocation = target.ClrVersions[0].LocalMatchingDac;

            if (args.Length == 3)
            {
                dacLocation = args[2];
            }

            if (!System.IO.File.Exists(dacLocation))
            {
                Console.WriteLine($"ERROR: Dac Location {dacLocation} does not exist, please point to DAC associated with the dump, you can also try your local DAC (%WINDIR%\\Microsoft.NET\\Framework64\\v4.0.30319\\mscordacwks.dll)");
            }

            var runtime = target.ClrVersions[0].CreateRuntime(dacLocation, ignoreMismatch: true);

            var heap = runtime.Heap;
            if (!heap.CanWalkHeap)
            {
                LogErrorWithTimeStamp("Heap is not walkable. Please collect a new dump.");
                return;
            }

            LogInfoWithTimeStamp("Crash Dump loaded.");

            var perTypeCounts = new Dictionary<ClrType, int>(1000000);

            var fileName = args[1];

            if (System.IO.File.Exists(fileName))
            {
                System.IO.File.Delete(fileName);
            }

            LogInfoWithTimeStamp("Creating SQLite Database filename: " + fileName + " ...");

            sqlite3* db;
            int error;
            if ((error = NativeMethods.sqlite3_open(fileName, out db)) != 0)
            {
                LogErrorWithTimeStamp("sqlite3_open failed with error code: " + error);
            }

            LogInfoWithTimeStamp("Successfully created SQLite Database filename: " + fileName);

            LogInfoWithTimeStamp("Creating SQLite Tables ...");

            if (!CreateTables(db))
            {
                return;
            }

            LogInfoWithTimeStamp("Successfully created SQLite Tables.");

            error = NativeMethods.sqlite3_exec(db, "BEGIN TRANSACTION;", IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
            if (error != 0)
            {
                LogErrorWithTimeStamp("sqlite3_exec -> BEGIN TRANSACTION; failed to execute with SQLite error code: " + error);
            }

            sqlite3_stmt* insertObjectsStmt, insertTypesStmt, insertRootsStmt, insertBlockingObjectsStmt;

            if (!PrepareInsertStatement(db, out insertTypesStmt, @"INSERT INTO Types(TypeIndex, Count, Size, Name) VALUES (@1, @2, @3, @4);"))
            {
                return;
            }

            if (!PrepareInsertStatement(db, out insertObjectsStmt, @"INSERT INTO Objects(ObjectId, TypeIndex, Size) VALUES (@1, @2, @3);"))
            {
                return;
            }

            if (!PrepareInsertStatement(db, out insertObjectReferencesStmt, @"INSERT INTO ObjectReferences(ObjectId, ObjectReference) VALUES (@1, @2);"))
            {
                return;
            }

            if (!PrepareInsertStatement(db, out insertRootsStmt, @"INSERT INTO Roots(TypeIndex, ObjectId, Address, AppDomainId, ManagedThreadId, IsInterior, IsPinned, IsPossibleFalsePositive, GCRootKind, Name) VALUES (@1, @2, @3, @4, @5, @6, @7, @8, @9, @10);"))
            {
                return;
            }

            if (!PrepareInsertStatement(db, out insertBlockingObjectsStmt, @"INSERT INTO BlockingObjects(ObjectId, Taken, RecursionCount, Owner, HasSingleOwner, ThreadOwnerIds, ThreadWaiterIds, BlockingReason) VALUES (@1, @2, @3, @4, @5, @6, @7, @8);"))
            {
                return;
            }

            LogInfoWithTimeStamp("Starting to populate Objects Table ...");

            long objectCount = 0;
            var objRefWalk = new Action<ulong, int>(ObjRefWalk);
            foreach (var obj in heap.EnumerateObjects())
            {
                objectCount++;
                if (objectCount % 1000000 == 0)
                {
                    LogInfoWithTimeStamp($"Enumerated objects: {objectCount}");
                }

                var type = heap.GetObjectType(obj.Address);
{'fraction_non_alphanumeric': 0.10865889472279078, 'fraction_numerical': 0.017709353030168195, 'mean_word_length': 2.0612912013075455, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '553946', 'n_tokens_mistral': 3306, 'n_tokens_neox': 3074, 'n_words': 730}