<commit_before>﻿using Devedse.DeveImagePyramid.Logging;
using System;
using DeveMultiCompressor.Config;
using System.IO;
using System.Linq;

namespace DeveMultiCompressor
{
    public class CompressorRunner
    {
        private ILogger _logger;
        private CompressorFinderFactory _compressionFinderFactory;

        public CompressorRunner(ILogger logger, CompressorFinderFactory compressorFinderFactory)
        {
            this._logger = logger;
            this._compressionFinderFactory = compressorFinderFactory;
        }

        public void GoCompress(CommandLineOptions options)
        {
            var outputDir = Path.Combine(FolderHelperMethods.AssemblyDirectory.Value, Constants.OutputDir);
            var allCompressors = _compressionFinderFactory.GetCompressors();

            var inputFile = new CompressorFileInfo(options.InputFile);

            if (options.UsePrecomp)
            {
                _logger.Write("Using precomp...");
                var preCompressor = _compressionFinderFactory.GetPreCompressor();
                var copiedFile = inputFile.CopyToDirectory(preCompressor.CompressorDir);
                inputFile = preCompressor.CompressFile(inputFile);
                copiedFile.Delete();
            }

            _logger.Write("Starting actual compression...");
            _logger.Write($"Handling input file: '{options.InputFile}'. Generating hash...");
            var hash = inputFile.GenerateHash();
            _logger.Write($"Generated hash of input file: {hash}");

            foreach (var compressor in allCompressors)
            {
                var newInputFile = inputFile.CopyToDirectory(compressor.CompressorDir);

                try
                {
                    var outputFile = compressor.CompressFile(newInputFile);
                    newInputFile.Delete();

                    if (options.Verify)
                    {
                        var expectedPath = newInputFile.FullPath;
                        var decompressedFile = compressor.DecompressFile(outputFile, expectedPath);
                        var decompressedFileHash = decompressedFile.GenerateHash();
                        if (hash != decompressedFileHash)
                        {
                            throw new Exception($"Hash of decompressed file: '{decompressedFile.FullPath}': '{decompressedFileHash}' does not match has of input file: '{inputFile.FullPath}': '{hash}'.");
                        }
                        else
                        {
                            _logger.Write($"File verified. Hash is equal to input file: {decompressedFileHash}", color: ConsoleColor.Green);
                        }
                    }

                    outputFile.MoveToDirectory(outputDir);
                    _logger.Write($"File compressed to '{outputFile.FileName}'. Size: {outputFile.GetFileSize()}. Hash: {outputFile.GenerateHash()}", color: ConsoleColor.Green);
                }
                catch (Exception ex)
                {
                    _logger.WriteError($"Error occured when compressing with {compressor.CompressorDir}. Error: {ex.ToString()}");
                }
            }

            if (options.UsePrecomp)
            {
                //If we use precomp, delete the .pcf file
                inputFile.Delete();
            }

            _logger.Write("Completed compression :)");
        }

        public void GoDecompress(CommandLineOptions options)
        {
            var lastPathOfOutputDir = Path.GetFileNameWithoutExtension(options.InputFile);
            var outputDir = Path.Combine(FolderHelperMethods.AssemblyDirectory.Value, Constants.OutputDir, lastPathOfOutputDir);
            var allCompressors = _compressionFinderFactory.GetCompressors();

            var inputFile = new CompressorFileInfo(options.InputFile);

            var inputFileExtension = Path.GetExtension(inputFile.FullPath);
            var validDecompressors = allCompressors.Where(t => string.Equals(inputFileExtension, "." + t.CompressorConfig.CompressedFileExtension, StringComparison.OrdinalIgnoreCase) || string.Equals(inputFileExtension, t.CompressorConfig.CompressedFileExtension, StringComparison.OrdinalIgnoreCase)).ToList();

            if (validDecompressors.Count == 0)
            {
                _logger.WriteError("No valid decompressor found, exiting...");
                return;
            }
            if (validDecompressors.Count > 1)
            {
                _logger.Write($"Found {validDecompressors.Count} decompressors. Taking first.", LogLevel.Warning, ConsoleColor.Yellow);
            }

            var firstDecompressor = validDecompressors.First();

            firstDecompressor.DecompressFileToDir(inputFile, outputDir);

            if (options.UsePrecomp)
            {
                var precomp = _compressionFinderFactory.GetPreCompressor();

                foreach (var file in Directory.GetFiles(outputDir))
                {
                    var extension = Path.GetExtension(file);
                    if (string.Equals(extension, "." + precomp.CompressorConfig.CompressedFileExtension, StringComparison.OrdinalIgnoreCase) || string.Equals(extension, precomp.CompressorConfig.CompressedFileExtension, StringComparison.OrdinalIgnoreCase))
                    {
                        var precompFile = new CompressorFileInfo(file);
                        var lastPartOfPrecompFile = Path.GetFileNameWithoutExtension(precompFile.FullPath);
                        var outputDirPrecomp = Path.Combine(outputDir, lastPartOfPrecompFile);
                        if (Directory.Exists(outputDirPrecomp))
                        {
                            _logger.Write($"Output directory '{outputDirPrecomp}' already exist. Possible file collisions...", LogLevel.Warning, ConsoleColor.Yellow);
                        }
                        precomp.DecompressFileToDir(precompFile, outputDirPrecomp);
                    }
                }
            }
        }
    }
}
<commit_msg>Added verification for Precomp<commit_after>﻿using Devedse.DeveImagePyramid.Logging;
using System;
using DeveMultiCompressor.Config;
using System.IO;
using System.Linq;

namespace DeveMultiCompressor
{
    public class CompressorRunner
    {
        private ILogger _logger;
        private CompressorFinderFactory _compressionFinderFactory;

        public CompressorRunner(ILogger logger, CompressorFinderFactory compressorFinderFactory)
        {
            this._logger = logger;
            this._compressionFinderFactory = compressorFinderFactory;
        }

        public void GoCompress(CommandLineOptions options)
        {
            var outputDir = Path.Combine(FolderHelperMethods.AssemblyDirectory.Value, Constants.OutputDir);
            var allCompressors = _compressionFinderFactory.GetCompressors();

            var inputFile = new CompressorFileInfo(options.InputFile);

            _logger.Write($"Handling input file: '{options.InputFile}'. Generating hash...");
            var hash = inputFile.GenerateHash();
            _logger.Write($"Generated hash of input file: {hash}");

            if (options.UsePrecomp)
            {
                _logger.Write("Using precomp...");
                var preCompressor = _compressionFinderFactory.GetPreCompressor();
                var copiedFile = inputFile.CopyToDirectory(preCompressor.CompressorDir);
                inputFile = preCompressor.CompressFile(inputFile);
                copiedFile.Delete();

                if (options.Verify)
                {
                    var expectedPath = copiedFile.FullPath;
                    var decompressedFile = preCompressor.DecompressFile(inputFile, expectedPath);
                    var decompressedFileHash = decompressedFile.GenerateHash();
                    if (hash != decompressedFileHash)
                    {
                        throw new Exception($"Hash of decompressed file (with precomp): '{decompressedFile.FullPath}': '{decompressedFileHash}' does not match has of input file: '{inputFile.FullPath}': '{hash}'.");
                    }
                    else
                    {
                        _logger.Write($"File verified (with precomp). Hash is equal to input file: {decompressedFileHash}", color: ConsoleColor.Green);
                    }
                }

                _logger.Write($"Generating hash of precomp file: '{inputFile.FullPath}'...");
                hash = inputFile.GenerateHash();
                _logger.Write($"Generated hash of precomp input file: {hash}");
            }

            _logger.Write("Starting actual compression...");


            foreach (var compressor in allCompressors)
            {
                var newInputFile = inputFile.CopyToDirectory(compressor.CompressorDir);

                try
                {
                    var outputFile = compressor.CompressFile(newInputFile);
                    newInputFile.Delete();

                    if (options.Verify)
                    {
                        var expectedPath = newInputFile.FullPath;
                        var decompressedFile = compressor.DecompressFile(outputFile, expectedPath);
                        var decompressedFileHash = decompressedFile.GenerateHash();
                        if (hash != decompressedFileHash)
                        {
                            throw new Exception($"Hash of decompressed file: '{decompressedFile.FullPath}': '{decompressedFileHash}' does not match has of input file: '{inputFile.FullPath}': '{hash}'.");
                        }
                        else
                        {
                            _logger.Write($"File verified. Hash is equal to input file: {decompressedFileHash}", color: ConsoleColor.Green);
                        }
                    }

                    outputFile.MoveToDirectory(outputDir);
                    _logger.Write($"File compressed to '{outputFile.FileName}'. Size: {outputFile.GetFileSize()}. Hash: {outputFile.GenerateHash()}", color: ConsoleColor.Green);
                }
                catch (Exception ex)
                {
                    _logger.WriteError($"Error occured when compressing with {compressor.CompressorDir}. Error: {ex.ToString()}");
                }
            }

            if (options.UsePrecomp)
            {
                //If we use precomp, delete the .pcf file
                inputFile.Delete();
            }

            _logger.Write("Completed compression :)");
        }

        public void GoDecompress(CommandLineOptions options)
        {
            var lastPathOfOutputDir = Path.GetFileNameWithoutExtension(options.InputFile);
            var outputDir = Path.Combine(FolderHelperMethods.AssemblyDirectory.Value, Constants.OutputDir, lastPathOfOutputDir);
            var allCompressors = _compressionFinderFactory.GetCompressors();

            var inputFile = new CompressorFileInfo(options.InputFile);

            var inputFileExtension = Path.GetExtension(inputFile.FullPath);
            var validDecompressors = allCompressors.Where(t => string.Equals(inputFileExtension, "." + t.CompressorConfig.CompressedFileExtension, StringComparison.OrdinalIgnoreCase) || string.Equals(inputFileExtension, t.CompressorConfig.CompressedFileExtension, StringComparison.OrdinalIgnoreCase)).ToList();

            if (validDecompressors.Count == 0)
            {
                _logger.WriteError("No valid decompressor found, exiting...");
                return;
            }
            if (validDecompressors.Count > 1)
            {
                _logger.Write($"Found {validDecompressors.Count} decompressors. Taking first.", LogLevel.Warning, ConsoleColor.Yellow);
            }

            var firstDecompressor = validDecompressors.First();

            firstDecompressor.DecompressFileToDir(inputFile, outputDir);

            if (options.UsePrecomp)
            {
                var precomp = _compressionFinderFactory.GetPreCompressor();

                foreach (var file in Directory.GetFiles(outputDir))
                {
                    var extension = Path.GetExtension(file);
                    if (string.Equals(extension, "." + precomp.CompressorConfig.CompressedFileExtension, StringComparison.OrdinalIgnoreCase) || string.Equals(extension, precomp.CompressorConfig.CompressedFileExtension, StringComparison.OrdinalIgnoreCase))
                    {
                        var precompFile = new CompressorFileInfo(file);
                        var lastPartOfPrecompFile = Path.GetFileNameWithoutExtension(precompFile.FullPath);
                        var outputDirPrecomp = Path.Combine(outputDir, lastPartOfPrecompFile);
                        if (Directory.Exists(outputDirPrecomp))
                        {
                            _logger.Write($"Output directory '{outputDirPrecomp}' already exist. Possible file collisions...", LogLevel.Warning, ConsoleColor.Yellow);
                        }
                        precomp.DecompressFileToDir(precompFile, outputDirPrecomp);
                    }
                }
            }
        }
    }
}

{'fraction_non_alphanumeric': 0.08501440922190202, 'fraction_numerical': 0.0003033520400424693, 'mean_word_length': 1.8950603732162459, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3091063', 'n_tokens_mistral': 3220, 'n_tokens_neox': 3018, 'n_words': 602}