<commit_before>﻿using System.IO;
using ICSharpCode.SharpZipLib.Tar;
using Serilog;

namespace Arkivverket.Arkade.Util
{
    /// <summary>
    ///  Using SharpZipLib for tar operations
    /// </summary>
    public class TarCompressionUtility : ICompressionUtility
    {
        private readonly ILogger _log = Log.ForContext<TarCompressionUtility>();

        public void ExtractFolderFromArchive(FileInfo file, DirectoryInfo targetDirectory)
        {
            Stream inStream = File.OpenRead(file.FullName);
            var tarArchive = TarArchive.CreateInputTarArchive(inStream);
            tarArchive.ExtractContents(targetDirectory.FullName);
            tarArchive.Close();
            inStream.Close();

            _log.Debug($"Extracted tar file: {file} to folder {targetDirectory}", file.FullName, targetDirectory.FullName);
        }

<commit_msg>Omit single root directory on un-taring input packages<commit_after>﻿using System.IO;
using System;
using ICSharpCode.SharpZipLib.Tar;
using Serilog;

namespace Arkivverket.Arkade.Util
{
    /// <summary>
    ///  Using SharpZipLib for tar operations
    /// </summary>
    public class TarCompressionUtility : ICompressionUtility
    {
        private readonly ILogger _log = Log.ForContext<TarCompressionUtility>();

        public void ExtractFolderFromArchive(FileInfo file, DirectoryInfo targetDirectory)
        {
            using (var inputStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read))
            {
                DirectoryInfo singleRootDirectory = GetSingleRootDirectory(inputStream);
                inputStream.Position = 0; // Needs resetting after GetSingleRootDirectory()

                var tarInputStream = new TarInputStream(inputStream);

                TarEntry tarEntry;
                while ((tarEntry = tarInputStream.GetNextEntry()) != null)
                {
                    if (tarEntry.IsDirectory)
                        continue;

                    string name = tarEntry.Name.Replace('/', Path.DirectorySeparatorChar);

                    if (singleRootDirectory != null)
                        name = name.Substring(singleRootDirectory.Name.Length);

                    if (Path.IsPathRooted(name))
                        name = name.Substring(Path.GetPathRoot(name).Length);

                    string fullName = Path.Combine(targetDirectory.FullName, name);
                    string directoryName = Path.GetDirectoryName(fullName);
                    if (directoryName != null) Directory.CreateDirectory(directoryName);

                    var outputStream = new FileStream(fullName, FileMode.Create);

                    tarInputStream.CopyEntryContents(outputStream);
                    outputStream.Close();

                    DateTime dateTime = DateTime.SpecifyKind(tarEntry.ModTime, DateTimeKind.Utc);
                    File.SetLastWriteTime(fullName, dateTime);
                }
                tarInputStream.Close();
            }
            _log.Debug($"Extracted tar file: {file} to folder {targetDirectory}", file.FullName, targetDirectory.FullName);
        }

        private static DirectoryInfo GetSingleRootDirectory(Stream inputStream)
        {
            var tarInputStream = new TarInputStream(inputStream);
            TarEntry firstEntry = tarInputStream.GetNextEntry();

            if (!firstEntry.IsDirectory)
                return null;

            TarEntry tarEntry;
            while ((tarEntry = tarInputStream.GetNextEntry()) != null)
                if (!tarEntry.Name.StartsWith(firstEntry.Name))
                    return null;

            return new DirectoryInfo(firstEntry.Name);
        }

{'fraction_non_alphanumeric': 0.08500814774579032, 'fraction_numerical': 0.00027159152634437803, 'mean_word_length': 2.18598615916955, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4224873', 'n_tokens_mistral': 936, 'n_tokens_neox': 881, 'n_words': 192}