<commit_before>﻿using System;
using System.CommandLine;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using SJP.Schematic.Core.Extensions;
using SJP.Schematic.Reporting;

namespace SJP.Schematic.Tool.Handlers
{
    internal sealed class ReportCommandHandler : DatabaseCommandHandler
    {
        public ReportCommandHandler(FileInfo filePath)
            : base(filePath)
        {
            ConfigureDotPath();
        }

        private void ConfigureDotPath()
        {
            var dotPath = Configuration.GetValue<string>("Graphviz:Dot");
            if (!dotPath.IsNullOrWhiteSpace())
                Environment.SetEnvironmentVariable("SCHEMATIC_GRAPHVIZ_DOT", dotPath, EnvironmentVariableTarget.Process);
        }

        public async Task<int> HandleCommandAsync(IConsole console, DirectoryInfo outputPath, CancellationToken cancellationToken)
        {
            var connection = GetSchematicConnection();
            var database = await connection.Dialect.GetRelationalDatabaseAsync(connection, cancellationToken).ConfigureAwait(false);

            var reportGenerator = new ReportGenerator(connection, database, outputPath);

            await reportGenerator.GenerateAsync(cancellationToken).ConfigureAwait(false);

            console.Out.Write("Report generated to: " + outputPath.FullName);
            return ErrorCode.Success;
        }
    }
}<commit_msg>When generating a report, use a snapshot DB to avoid keeping all data in memory.<commit_after>﻿using System;
using System.CommandLine;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using AutoMapper;
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Configuration;
using SJP.Schematic.Core;
using SJP.Schematic.Core.Extensions;
using SJP.Schematic.Reporting;
using SJP.Schematic.Reporting.Snapshot;
using SJP.Schematic.Serialization.Mapping;
using SJP.Schematic.Sqlite;

namespace SJP.Schematic.Tool.Handlers
{
    internal sealed class ReportCommandHandler : DatabaseCommandHandler
    {
        public ReportCommandHandler(FileInfo filePath)
            : base(filePath)
        {
            ConfigureDotPath();
        }

        public async Task<int> HandleCommandAsync(IConsole console, DirectoryInfo outputPath, CancellationToken cancellationToken)
        {
            var connection = GetSchematicConnection();
            var database = await connection.Dialect.GetRelationalDatabaseAsync(connection, cancellationToken).ConfigureAwait(false);

            if (!outputPath.Exists)
                outputPath.Create();

            var snapshotDbPath = Path.Combine(outputPath.FullName, $"snapshot-{ Guid.NewGuid() }.db");
            var snapshotDb = await CreateSnapshotRelationalDatabaseAsync(snapshotDbPath, database).ConfigureAwait(false);

            var reportGenerator = new ReportGenerator(connection, snapshotDb, outputPath);
            await reportGenerator.GenerateAsync(cancellationToken).ConfigureAwait(false);

            if (File.Exists(snapshotDbPath))
                File.Delete(snapshotDbPath);

            console.Out.Write("Report generated to: " + outputPath.FullName);
            return ErrorCode.Success;
        }

        private static async Task<IRelationalDatabase> CreateSnapshotRelationalDatabaseAsync(string snapshotDbPath, IRelationalDatabase database)
        {
            var snapshotConnection = CreateSnapshotConnection(snapshotDbPath);
            var mapper = CreateMapper();

            var schema = new SnapshotSchema(snapshotConnection.DbConnection);
            await schema.EnsureSchemaExistsAsync().ConfigureAwait(false);

            // snapshotting
            var objectWriter = new SnapshotRelationalDatabaseWriter(snapshotConnection.DbConnection, mapper);
            await objectWriter.SnapshotDatabaseObjectsAsync(database).ConfigureAwait(false);

            // reading snapshot
            return new SnapshotRelationalDatabaseReader(snapshotConnection.DbConnection, mapper);
        }

        private void ConfigureDotPath()
        {
            var dotPath = Configuration.GetValue<string>("Graphviz:Dot");
            if (!dotPath.IsNullOrWhiteSpace())
                Environment.SetEnvironmentVariable("SCHEMATIC_GRAPHVIZ_DOT", dotPath, EnvironmentVariableTarget.Process);
        }

        private static IMapper CreateMapper()
        {
            var config = new MapperConfiguration(config => config.AddMaps(typeof(RelationalDatabaseTableProfile).Assembly));
            config.AssertConfigurationIsValid();
            return new Mapper(config);
        }

        private static ISchematicConnection CreateSnapshotConnection(string snapshotDbPath)
        {
            var builder = new SqliteConnectionStringBuilder { DataSource = snapshotDbPath };

            var connectionFactory = new SqliteConnectionFactory(builder.ToString());
            var dialect = new SqliteDialect();

            return new SchematicConnection(connectionFactory, dialect);
        }
    }
}
{'fraction_non_alphanumeric': 0.07811260904044409, 'fraction_numerical': 0.0, 'mean_word_length': 3.169421487603306, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3951948', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1222, 'n_words': 253}