<commit_before>﻿using Dangl.WebDocumentation.Dtos;
using Dangl.WebDocumentation.Models;
using Microsoft.AspNetCore.StaticFiles;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Dangl.WebDocumentation.Services
{
    public class AzureBlobStorageProjectFilesService : IProjectFilesService
    {
        private readonly ApplicationDbContext _context;
        private readonly AspNetCore.FileHandling.IFileManager _fileManager;
        private static readonly FileExtensionContentTypeProvider _fileExtensionContentTypeProvider = new FileExtensionContentTypeProvider();

        public AzureBlobStorageProjectFilesService(ApplicationDbContext context,
            Dangl.AspNetCore.FileHandling.IFileManager fileManager)
        {
            _context = context;
            _fileManager = fileManager;
        }

        public Task<string> GetEntryFilePathForProject(string projectName)
        {
            var entryPath = _context.DocumentationProjects
                .Where(p => p.Name == projectName)
                .Select(p => p.PathToIndex)
                .FirstAsync();
            return entryPath;
        }

        public async Task<ProjectFileDto> GetFileForProject(string projectName, string version, string filePath)
        {
            var projectId = await _context.DocumentationProjects
                .Where(p => p.Name == projectName)
                .Select(p => new { p.Id })
                .FirstOrDefaultAsync();
            if (projectId == null)
            {
                return null;
            }
            var versionId = await _context.DocumentationProjectVersionss
                .Where(v => v.ProjectName == projectName && v.Version == version)
                .Select(v => new { v.FileId })
                .FirstOrDefaultAsync();
            if (versionId == null)
            {
                return null;
            }

            var blobFilePath = GetFilePath(projectId.Id, versionId.FileId, filePath);
            var fileResult = await _fileManager.GetFileAsync(AppConstants.PROJECTS_CONTAINER, blobFilePath);
            if (!fileResult.IsSuccess)
            {
                return null;
            }

            return new ProjectFileDto
            {
                MimeType = GetMimeTypeForFilePath(filePath),
                FileStream = fileResult.Value
            };
        }

        private string GetMimeTypeForFilePath(string filePath)
        {
            var extension = Path.GetExtension(filePath);
            if (_fileExtensionContentTypeProvider.Mappings.ContainsKey(extension))
            {
                return _fileExtensionContentTypeProvider.Mappings[extension];
            }
            return "application/octet-stream";
        }

        public async Task<bool> UploadProjectPackageAsync(string projectName, string version, Stream zipArchiveStream)
        {
            var projectId = await _context.DocumentationProjects
                .Where(p => p.Name == projectName)
                .Select(p => p.Id)
                .FirstAsync();
            // Try to read as zip file
            try
            {
                using (var transaction = await _context.Database.BeginTransactionAsync())
                {
                    var newVersion = new DocumentationProjectVersion
                    {
                        ProjectName = projectName,
                        Version = version
                    };
                    _context.DocumentationProjectVersionss.Add(newVersion);
                    await _context.SaveChangesAsync();

                    await SavedZipFileContentsToBlobStorage(projectId, newVersion.FileId, zipArchiveStream);

                    transaction.Commit();
                    return true;
                }
            }
            catch (InvalidDataException)
            {
                // Could not read the file as zip
                return false;
            }
            catch
            {
                return false;
            }
        }

        private async Task SavedZipFileContentsToBlobStorage(Guid projectId, Guid versionId, Stream zipArchiveStream)
        {
            List<string> savedFilePaths = new List<string>();
            using (var zipArchive = new ZipArchive(zipArchiveStream))
            {
                foreach (var archiveEntry in zipArchive.Entries
                    .Where(e => e.Length > 0)) // Length == 0 means it's just a folder
                {
                    using (var entryStream = archiveEntry.Open())
                    {
                        var filePath = GetFilePath(projectId, versionId, archiveEntry.FullName);
                        savedFilePaths.Add(filePath);
                        await _fileManager.SaveFileAsync(AppConstants.PROJECTS_CONTAINER, filePath, entryStream).ConfigureAwait(false);
                    }
                }
            }

            var metadataFileSavePath = GetMetadataSavedFilesPath(projectId, versionId);
            savedFilePaths.Add(metadataFileSavePath);
            using (var memStream = new MemoryStream())
            {
                using (var streamWriter = new StreamWriter(memStream, Encoding.UTF8, 4096, true))
                {
                    var metadataArray = JsonConvert.SerializeObject(savedFilePaths);
                    await streamWriter.WriteAsync(metadataArray);
                }

                memStream.Position = 0;
                await _fileManager.SaveFileAsync(AppConstants.PROJECTS_CONTAINER, metadataFileSavePath, memStream);
            }
        }

        // This is a bit hacky, but since there's no real concept of directories or sub folders
        // in Azure blob storage, we're just saving a metadata file of all the file paths belonging
        // to a specific project / version combination so we can easily delete it later
        private string GetMetadataSavedFilesPath(Guid projectId, Guid versionId)
        {
            return GetFilePath(projectId, versionId, "METADATA_SAVED_FILES.json");
        }

        public async Task<bool> DeleteProjectVersionPackageAsync(Guid projectId, string version)
        {
            var projectVersion = await _context.DocumentationProjectVersionss
                .FirstOrDefaultAsync(v => v.Version == version && v.Project.Id == projectId);
            if (projectVersion == null)
            {
                return false;
            }

            var metadataPath = GetMetadataSavedFilesPath(projectId, projectVersion.FileId);

            try
            {
                using (var metadataStream = (await _fileManager.GetFileAsync(AppConstants.PROJECTS_CONTAINER, metadataPath)).Value)
                {
                    using (var streamReader = new StreamReader(metadataStream))
                    {
                        var json = await streamReader.ReadToEndAsync();
                        var filePaths = JsonConvert.DeserializeObject<List<string>>(json);
                        foreach (var filePath in filePaths)
                        {
                            await _fileManager.DeleteFileAsync(AppConstants.PROJECTS_CONTAINER, filePath);
                        }
                        _context.DocumentationProjectVersionss.Remove(projectVersion);
                        await _context.SaveChangesAsync();
                        return true;
                    }
                }
            }
            catch
            {
                return false;
            }
        }

        private string GetFilePath(Guid projectId, Guid versionId, string filePath)
        {
            filePath = filePath.Replace('/', '\\');
            return Path.Combine(projectId.ToString(), versionId.ToString(), filePath);
        }
    }
}<commit_msg>Refactor<commit_after>﻿using Dangl.AspNetCore.FileHandling;
using Dangl.WebDocumentation.Dtos;
using Dangl.WebDocumentation.Models;
using Microsoft.AspNetCore.StaticFiles;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace Dangl.WebDocumentation.Services
{
    public class AzureBlobStorageProjectFilesService : IProjectFilesService
    {
        private readonly ApplicationDbContext _context;
        private readonly AspNetCore.FileHandling.IFileManager _fileManager;
        private static readonly FileExtensionContentTypeProvider _fileExtensionContentTypeProvider = new FileExtensionContentTypeProvider();

        public AzureBlobStorageProjectFilesService(ApplicationDbContext context,
            Dangl.AspNetCore.FileHandling.IFileManager fileManager)
        {
            _context = context;
            _fileManager = fileManager;
        }

        public Task<string> GetEntryFilePathForProject(string projectName)
        {
            var entryPath = _context.DocumentationProjects
                .Where(p => p.Name == projectName)
                .Select(p => p.PathToIndex)
                .FirstAsync();
            return entryPath;
        }

        public async Task<ProjectFileDto> GetFileForProject(string projectName, string version, string filePath)
        {
            var projectId = await _context.DocumentationProjects
                .Where(p => p.Name == projectName)
                .Select(p => new { p.Id })
                .FirstOrDefaultAsync();
            if (projectId == null)
            {
                return null;
            }
            var versionId = await _context.DocumentationProjectVersionss
                .Where(v => v.ProjectName == projectName && v.Version == version)
                .Select(v => new { v.FileId })
                .FirstOrDefaultAsync();
            if (versionId == null)
            {
                return null;
            }

            var blobFilePath = GetFilePath(projectId.Id, versionId.FileId, filePath);
            var fileResult = await _fileManager.GetFileAsync(AppConstants.PROJECTS_CONTAINER, blobFilePath);
            if (!fileResult.IsSuccess)
            {
                return null;
            }

            return new ProjectFileDto
            {
                MimeType = GetMimeTypeForFilePath(filePath),
                FileStream = fileResult.Value
            };
        }

        private string GetMimeTypeForFilePath(string filePath)
        {
            var extension = Path.GetExtension(filePath);
            if (_fileExtensionContentTypeProvider.Mappings.ContainsKey(extension))
            {
                return _fileExtensionContentTypeProvider.Mappings[extension];
            }
            return "application/octet-stream";
        }

        public async Task<bool> UploadProjectPackageAsync(string projectName, string version, Stream zipArchiveStream)
        {
            var projectId = await _context.DocumentationProjects
                .Where(p => p.Name == projectName)
                .Select(p => p.Id)
                .FirstAsync();
            // Try to read as zip file
            try
            {
                using (var transaction = await _context.Database.BeginTransactionAsync())
                {
                    var newVersion = new DocumentationProjectVersion
                    {
                        ProjectName = projectName,
                        Version = version
                    };
                    _context.DocumentationProjectVersionss.Add(newVersion);
                    await _context.SaveChangesAsync();

                    await SavedZipFileContentsToBlobStorage(projectId, newVersion.FileId, zipArchiveStream, _fileManager);

                    transaction.Commit();
                    return true;
                }
            }
            catch (InvalidDataException)
            {
                // Could not read the file as zip
                return false;
            }
            catch
            {
                return false;
            }
        }

        public static async Task SavedZipFileContentsToBlobStorage(Guid projectId, Guid versionId, Stream zipArchiveStream, IFileManager fileManager)
        {
            var savedFilePaths = new List<string>();
            using (var zipArchive = new ZipArchive(zipArchiveStream))
            {
                var entriesToUpload = zipArchive.Entries
                    .Where(e => e.Length > 0) // Length == 0 means it's just a folder
                    .ToList();

                using (var semaphore = new SemaphoreSlim(16))
                {
                    var uploadTasks = entriesToUpload.Select(async entry =>
                    {
                        await semaphore.WaitAsync().ConfigureAwait(false);
                        try
                        {
                            using (var entryStream = entry.Open())
                            {
                                var filePath = GetFilePath(projectId, versionId, entry.FullName);
                                savedFilePaths.Add(filePath);
                                await fileManager.SaveFileAsync(AppConstants.PROJECTS_CONTAINER, filePath, entryStream).ConfigureAwait(false);
                            }
                        }
                        finally
                        {
                            semaphore.Release();
                        }
                    });
                    await Task.WhenAll(uploadTasks).ConfigureAwait(false);
                }
            }


            var metadataFileSavePath = GetMetadataSavedFilesPath(projectId, versionId);
            savedFilePaths.Add(metadataFileSavePath);
            using (var memStream = new MemoryStream())
            {
                using (var streamWriter = new StreamWriter(memStream, Encoding.UTF8, 4096, true))
                {
                    var metadataArray = JsonConvert.SerializeObject(savedFilePaths);
                    await streamWriter.WriteAsync(metadataArray);
                }

                memStream.Position = 0;
                await fileManager.SaveFileAsync(AppConstants.PROJECTS_CONTAINER, metadataFileSavePath, memStream);
            }
        }

        // This is a bit hacky, but since there's no real concept of directories or sub folders
        // in Azure blob storage, we're just saving a metadata file of all the file paths belonging
        // to a specific project / version combination so we can easily delete it later
        private static string GetMetadataSavedFilesPath(Guid projectId, Guid versionId)
        {
            return GetFilePath(projectId, versionId, "METADATA_SAVED_FILES.json");
        }

        public async Task<bool> DeleteProjectVersionPackageAsync(Guid projectId, string version)
        {
            var projectVersion = await _context.DocumentationProjectVersionss
                .FirstOrDefaultAsync(v => v.Version == version && v.Project.Id == projectId);
            if (projectVersion == null)
            {
                return false;
            }

            var metadataPath = GetMetadataSavedFilesPath(projectId, projectVersion.FileId);

            try
            {
                using (var metadataStream = (await _fileManager.GetFileAsync(AppConstants.PROJECTS_CONTAINER, metadataPath)).Value)
                {
                    using (var streamReader = new StreamReader(metadataStream))
                    {
                        var json = await streamReader.ReadToEndAsync();
                        var filePaths = JsonConvert.DeserializeObject<List<string>>(json);
                        foreach (var filePath in filePaths)
                        {
                            await _fileManager.DeleteFileAsync(AppConstants.PROJECTS_CONTAINER, filePath);
                        }
                        _context.DocumentationProjectVersionss.Remove(projectVersion);
                        await _context.SaveChangesAsync();
                        return true;
                    }
                }
            }
            catch
            {
                return false;
            }
        }

        private static string GetFilePath(Guid projectId, Guid versionId, string filePath)
        {
            filePath = filePath.Replace('/', '\\');
            return Path.Combine(projectId.ToString(), versionId.ToString(), filePath);
        }
    }
}
{'fraction_non_alphanumeric': 0.06989798998008089, 'fraction_numerical': 0.0010864972535763867, 'mean_word_length': 1.6107784431137724, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4629289', 'n_tokens_mistral': 3931, 'n_tokens_neox': 3729, 'n_words': 853}