<commit_before>﻿using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Web.Models;
using File = System.IO.File;

namespace Web.Engine.Services
{
    public interface IFileStorage
    {
        Task<(string Key, string IV, string Path)> Save(Stream stream);

        Task Delete(string path);
        void TryDelete(params string[] paths);

        FileMeta Open(string path, string extension, string key, string iv);
        Task<FileMeta> Open(int id);
    }

    public class FileStorage : IFileStorage
    {
        private readonly ApplicationDbContext _db;
        private readonly string _storageRoot;
        private readonly IFileEncryptor _encryptor;
        private readonly IFileDecoder _fileDecoder;

        public FileStorage(IOptions<DRSConfig> config, ApplicationDbContext db, IFileEncryptor encryptor, IFileDecoder fileDecoder)
        {
            if (config.Value == null)
            {
                throw new ArgumentNullException(nameof(config));
            }

            _db = db ?? throw new ArgumentNullException(nameof(db));
            _encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor));
            _fileDecoder = fileDecoder ?? throw new ArgumentNullException(nameof(fileDecoder));
            _storageRoot = config.Value.DocumentPath ?? throw new ArgumentNullException(nameof(config.Value.DocumentPath));
        }

        private (string Key, string IV) EncryptAccessKeys(byte[] key, byte[] iv)
            => (_encryptor.Encrypt(key), _encryptor.Encrypt(iv));

        private (byte[] Key, byte[] IV) DecryptAccessKeys(string key, string iv)
            => (_encryptor.DecryptBase64(key), _encryptor.DecryptBase64(iv));

        private async Task<int> GetNextDirectorySeed()
        {
            var any = await _db.Documents.AnyAsync();

            if (!any)
            {
                return 1;
            }

            var max = await _db.Documents.MaxAsync(d => d.Id);

            return ++max;
        }

        private FileMeta Open(string path, string extension, byte[] key, byte[] iv)
        {
            Stream streamCreeator() => _encryptor.Decrypt(File.OpenRead(path), key, iv);
            var contentType = MimeTypes.MimeTypeMap.GetMimeType(extension);

            return new FileMeta(streamCreeator, contentType, _fileDecoder.Get(path));
        }

        public async Task<FileMeta> Open(int id)
        {
            var dataFile = await _db.DataFiles
                .Where(df => df.Id == id)
                .Select(df => new
                {
                    df.Path,
                    df.Extension,
                    df.Key,
                    df.IV
                })
                .SingleAsync()
                .ConfigureAwait(false);

            return Open(dataFile.Path, dataFile.Extension, dataFile.Key, dataFile.IV);
        }

        public FileMeta Open(string path, string extension, string key, string iv)
        {
            var (Key, IV) = DecryptAccessKeys(key, iv);

            return Open(path, extension, Key, IV);
        }

        public async Task<(string Key, string IV, string Path)> Save(Stream stream)
        {
            var accessKeys = _encryptor.GenerateKeyAndIv();
            var path = await Save(stream, accessKeys.Key, accessKeys.IV);

            var (Key, IV) = EncryptAccessKeys(accessKeys.Key, accessKeys.IV);

            return (Key, IV, path);
        }

        private async Task<string> Save(Stream stream, byte[] key, byte[] iv)
        {
            var path = await GetNewFileName();

            await CreateDirectoryIfNotFound(Path.GetDirectoryName(path))
                .ConfigureAwait(false);

            using (var encrypted = _encryptor.Encrypt(stream, key, iv))
            {
                using (var output = File.Create(path))
                {
                    encrypted.CopyTo(output);
                    encrypted.Dispose();
                }
            }

            return path;
        }

        public void TryDelete(params string[] paths)
        {
            foreach (var path in paths)
            {
                try
                {
                    Delete(path)
                        .ConfigureAwait(false);
                }
                catch
                {
                }
            }
        }

        public async Task Delete(string path)
        {
            if (!File.Exists(path))
            {
                return;
            }

            await Task.Factory
                .StartNew(() => File.Delete(path))
                .ConfigureAwait(false);
        }

        private async Task<string> GetNewFileName()
            => GetNewFileName(await GetNextDirectorySeed(), _storageRoot);

        private static string GetNewFileName(int directorySeed, string rootPath)
        {
            var subFolder1 = Math.Floor(directorySeed / 1024m / 1024m / 1024m);
            var subFolder2 = Math.Floor(subFolder1 / 1024m / 1024m);
            var subFolder3 = Math.Floor(subFolder2 / 1024m);

            return Path.Combine(rootPath,
                $@"{subFolder1}\{subFolder2}\{subFolder3}\{Guid.NewGuid():N}.bin");
        }

        private Task CreateDirectoryIfNotFound(string dirPath)
        {
            if (Directory.Exists(dirPath))
            {
                return Task.FromResult(0);
            }

            return Task.Factory
                .StartNew(() => Directory.CreateDirectory(dirPath));
        }
    }
}<commit_msg>pass the correct info so we get the correct decoder<commit_after>﻿using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Web.Models;
using File = System.IO.File;

namespace Web.Engine.Services
{
    public interface IFileStorage
    {
        Task<(string Key, string IV, string Path)> Save(Stream stream);

        Task Delete(string path);
        void TryDelete(params string[] paths);

        FileMeta Open(string path, string extension, string key, string iv);
        Task<FileMeta> Open(int id);
    }

    public class FileStorage : IFileStorage
    {
        private readonly ApplicationDbContext _db;
        private readonly string _storageRoot;
        private readonly IFileEncryptor _encryptor;
        private readonly IFileDecoder _fileDecoder;

        public FileStorage(IOptions<DRSConfig> config, ApplicationDbContext db, IFileEncryptor encryptor, IFileDecoder fileDecoder)
        {
            if (config.Value == null)
            {
                throw new ArgumentNullException(nameof(config));
            }

            _db = db ?? throw new ArgumentNullException(nameof(db));
            _encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor));
            _fileDecoder = fileDecoder ?? throw new ArgumentNullException(nameof(fileDecoder));
            _storageRoot = config.Value.DocumentPath ?? throw new ArgumentNullException(nameof(config.Value.DocumentPath));
        }

        private (string Key, string IV) EncryptAccessKeys(byte[] key, byte[] iv)
            => (_encryptor.Encrypt(key), _encryptor.Encrypt(iv));

        private (byte[] Key, byte[] IV) DecryptAccessKeys(string key, string iv)
            => (_encryptor.DecryptBase64(key), _encryptor.DecryptBase64(iv));

        private async Task<int> GetNextDirectorySeed()
        {
            var any = await _db.Documents.AnyAsync();

            if (!any)
            {
                return 1;
            }

            var max = await _db.Documents.MaxAsync(d => d.Id);

            return ++max;
        }

        private FileMeta Open(string path, string extension, byte[] key, byte[] iv)
        {
            Stream streamCreeator() => _encryptor.Decrypt(File.OpenRead(path), key, iv);
            var contentType = MimeTypes.MimeTypeMap.GetMimeType(extension);

            return new FileMeta(streamCreeator, contentType, _fileDecoder.Get(extension));
        }

        public async Task<FileMeta> Open(int id)
        {
            var dataFile = await _db.DataFiles
                .Where(df => df.Id == id)
                .Select(df => new
                {
                    df.Path,
                    df.Extension,
                    df.Key,
                    df.IV
                })
                .SingleAsync()
                .ConfigureAwait(false);

            return Open(dataFile.Path, dataFile.Extension, dataFile.Key, dataFile.IV);
        }

        public FileMeta Open(string path, string extension, string key, string iv)
        {
            var (Key, IV) = DecryptAccessKeys(key, iv);

            return Open(path, extension, Key, IV);
        }

        public async Task<(string Key, string IV, string Path)> Save(Stream stream)
        {
            var accessKeys = _encryptor.GenerateKeyAndIv();
            var path = await Save(stream, accessKeys.Key, accessKeys.IV);

            var (Key, IV) = EncryptAccessKeys(accessKeys.Key, accessKeys.IV);

            return (Key, IV, path);
        }

        private async Task<string> Save(Stream stream, byte[] key, byte[] iv)
        {
            var path = await GetNewFileName();

            await CreateDirectoryIfNotFound(Path.GetDirectoryName(path))
                .ConfigureAwait(false);

            using (var encrypted = _encryptor.Encrypt(stream, key, iv))
            {
                using (var output = File.Create(path))
                {
                    encrypted.CopyTo(output);
                    encrypted.Dispose();
                }
            }

            return path;
        }

        public void TryDelete(params string[] paths)
        {
            foreach (var path in paths)
            {
                try
                {
                    Delete(path)
                        .ConfigureAwait(false);
                }
                catch
                {
                }
            }
        }

        public async Task Delete(string path)
        {
            if (!File.Exists(path))
            {
                return;
            }

            await Task.Factory
                .StartNew(() => File.Delete(path))
                .ConfigureAwait(false);
        }

        private async Task<string> GetNewFileName()
            => GetNewFileName(await GetNextDirectorySeed(), _storageRoot);

        private static string GetNewFileName(int directorySeed, string rootPath)
        {
            var subFolder1 = Math.Floor(directorySeed / 1024m / 1024m / 1024m);
            var subFolder2 = Math.Floor(subFolder1 / 1024m / 1024m);
            var subFolder3 = Math.Floor(subFolder2 / 1024m);

            return Path.Combine(rootPath,
                $@"{subFolder1}\{subFolder2}\{subFolder3}\{Guid.NewGuid():N}.bin");
        }

        private Task CreateDirectoryIfNotFound(string dirPath)
        {
            if (Directory.Exists(dirPath))
            {
                return Task.FromResult(0);
            }

            return Task.Factory
                .StartNew(() => Directory.CreateDirectory(dirPath));
        }
    }
}
{'fraction_non_alphanumeric': 0.09440528241277772, 'fraction_numerical': 0.006781475863299724, 'mean_word_length': 1.7230320699708455, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6631563', 'n_tokens_mistral': 3043, 'n_tokens_neox': 2848, 'n_words': 693}