<commit_before>﻿/*
 *      Copyright (c) 2014-2015, SynapLink, LLC
 *      All Rights reserved.
 *      
 *      Released under BSD 3-Clause licence. See terms in
 *      /LICENCE, or online: http://opensource.org/licenses/BSD-3-Clause
 */
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace SynapLink.Zener.Archives
{
    /// <summary>
    /// Represents a Microsoft Cabinet/MS-CAB archive.
    /// </summary>
    public sealed class CabinetArchive
        : Archive, IDisposable
    {
        private static readonly IEnumerable<byte> Signature;
        private const int
            ASCII_NUL               = 0x00,

            U1                      = 1,
            U2                      = 2,
            U4                      = 4,
            HEADER_MIN_LENGTH       = 36,
            // Mandatory header field offsets. These
            // header fields will always be present within
            // a cabinet file.
            SIGNATURE_OFFSET        = 0,
            SIGNATURE_LEN           = U4,

            SIZE_OFFSET             = 8,
            SIZE_LEN                = U4,

            FILE_OFFSET_OFFSET      = 16,
            FILE_OFFSET_LEN         = U4,

            VERSION_MIN_OFFSET      = 24,
            VERSION_MIN_LEN         = U1,
            VERSION_MAJ_OFFSET      = 25,
            VERSION_MAJ_LEN         = U1,

            VERSION_MAJ_SUPPORTED   = 1,
            VERSION_MIN_SUPPORTED   = 3,

            FOLDER_COUNT_OFFSET     = 26,
            FOLDER_COUNT_LEN        = U2,
            FILE_COUNT_OFFSET       = 28,
            FILE_COUNT_LEN          = U2,

            HEADER_FLAGS_OFFSET     = 30,
            HEADER_FLAGS_LEN        = U2,

            CABINET_SETID_OFFSET    = 32,
            CABINET_SETID_LEN       = U2,
            CABINET_SETLEN_OFFSET   = 34,
            CABINET_SETLEN_LEN      = U2,

            // The offset of the field containing
            // the length of the abReserve field in this
            // CFHEADER block.
            HDR_ABRLEN_OFFSET       = 36,
            HDR_ABRLEN_LEN          = U2,
            // The offset of the abReserve field in this
            // CFHEADER block.
            HDR_ABR_OFFSET          = 40,
            // The offset of the field containing the
            // length of the abReserve field in each
            // CFFOLDER block.
            FDR_ABRLEN_OFFSET       = 38,
            FDR_ABRLEN_LEN          = U1,
            // The offset of the field containing the
            // length of the abReserve field in each
            // CFDATA block.
            DATA_ABRLEN_OFFSET      = 39,
            DATA_ABRLEN_LEN         = U1
            ;

        /// <summary>
        /// Represents the flags within the
        /// cabinet file's header.
        /// </summary>
        [Flags]
        private enum CabinetHeaderFlags : ushort
        {
            /// <summary>
            /// Indicates that this cabinet file is not
            /// the first in a set of cabinet files.
            /// </summary>
            IsNotFirst      = 0x0001,
            /// <summary>
            /// Indicates that this cabinet file is not
            /// the last in a set of cabinet files.
            /// </summary>
            IsNotLast       = 0x0002,
            /// <summary>
            /// Indicates that the cabinet file contains
            /// the reserved fields cbCFHeader, cbCFFolder,
            /// and cbCFData in the current CFHEADER block.
            /// </summary>
            ReservedPresent = 0x0004
        }

        static CabinetArchive()
        {
            Signature = new byte[4] { 0x4d, 0x53, 0x43, 0x46 };
        }

        // The absolute offset of the first CFFILE block.
        private readonly uint _filesFirstOffset;
        private readonly ushort 
            // The number of files/folders stored within
            // the archive.
            _filesCount, _foldersCount,
            // The length of the CFHEADER block's abReserve
            // field.
            _headerAbLength;
        // The lengths of the abReserve fields in CFFOLDER
        // and CFDATA blocks, respectively.
        private readonly byte _folderAbLength, _dataAbLength;
        // Flags indicating the presence of optional fields
        // within the cabinet's main header.
        private readonly CabinetHeaderFlags _flags;

        // The names of the files stored within the archive.
        private List<string> _names;
        // A set of Filemark structs indicating the positions
        // of each file within the temporary file storing
        // their uncompressed representations.
        private List<Filemark> _marks;
        // A FileStream with the temporary file open and
        // ready for reading/writing.
        private FileStream _dataDump;
        // Object to use in locks when reading from the
        // _dataDump FileStream.
        private object _dataLock;

        /// <summary>
        /// Creates a new CabinetArchive.
        /// </summary>
        /// <param name="stream">The stream containing the archive's bytes.</param>
        /// <exception cref="System.ArgumentNullException">
        ///     Thrown when the provided stream is null.
        /// </exception>
        /// <exception cref="System.ArgumentException">
        ///     Thrown when the provided stream does not support the
        ///     required operations.
        /// </exception>
        /// <exception cref="System.IO.InvalidDataException">
        ///     Thrown when the stream's data does not pass verification.
        /// </exception>
        /// <exception cref="System.NotSupportedException">
        ///     Thrown when the file format version of the provided cabinet
        ///     file is not supported by the class.
        /// </exception>
        public CabinetArchive(Stream stream)
        {
            if (stream == null)
                throw new ArgumentNullException(
                    "The provided stream cannot be null.",
                    "stream"
                    );

            if (!stream.CanSeek)
                throw new ArgumentException(
                    "The provided stream does not support seeking."
                    );

            if (!stream.CanRead)
                throw new ArgumentException(
                    "The provided stream does not support reading."
                    );

            stream.Position = 0;

            // We know that these headers will be present
            // in every cabinet file, so we can just read
            // them.
            byte[] hdrBuf = new byte[HEADER_MIN_LENGTH];
            stream.Read(hdrBuf, 0, HEADER_MIN_LENGTH);<commit_msg>* initialise new private fields<commit_after>﻿/*
 *      Copyright (c) 2014-2015, SynapLink, LLC
 *      All Rights reserved.
 *      
 *      Released under BSD 3-Clause licence. See terms in
 *      /LICENCE, or online: http://opensource.org/licenses/BSD-3-Clause
 */
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace SynapLink.Zener.Archives
{
    /// <summary>
    /// Represents a Microsoft Cabinet/MS-CAB archive.
    /// </summary>
    public sealed class CabinetArchive
        : Archive, IDisposable
    {
        private static readonly IEnumerable<byte> Signature;
        private const int
            ASCII_NUL               = 0x00,

            U1                      = 1,
            U2                      = 2,
            U4                      = 4,
            HEADER_MIN_LENGTH       = 36,
            // Mandatory header field offsets. These
            // header fields will always be present within
            // a cabinet file.
            SIGNATURE_OFFSET        = 0,
            SIGNATURE_LEN           = U4,

            SIZE_OFFSET             = 8,
            SIZE_LEN                = U4,

            FILE_OFFSET_OFFSET      = 16,
            FILE_OFFSET_LEN         = U4,

            VERSION_MIN_OFFSET      = 24,
            VERSION_MIN_LEN         = U1,
            VERSION_MAJ_OFFSET      = 25,
            VERSION_MAJ_LEN         = U1,

            VERSION_MAJ_SUPPORTED   = 1,
            VERSION_MIN_SUPPORTED   = 3,

            FOLDER_COUNT_OFFSET     = 26,
            FOLDER_COUNT_LEN        = U2,
            FILE_COUNT_OFFSET       = 28,
            FILE_COUNT_LEN          = U2,

            HEADER_FLAGS_OFFSET     = 30,
            HEADER_FLAGS_LEN        = U2,

            CABINET_SETID_OFFSET    = 32,
            CABINET_SETID_LEN       = U2,
            CABINET_SETLEN_OFFSET   = 34,
            CABINET_SETLEN_LEN      = U2,

            // The offset of the field containing
            // the length of the abReserve field in this
            // CFHEADER block.
            HDR_ABRLEN_OFFSET       = 36,
            HDR_ABRLEN_LEN          = U2,
            // The offset of the abReserve field in this
            // CFHEADER block.
            HDR_ABR_OFFSET          = 40,
            // The offset of the field containing the
            // length of the abReserve field in each
            // CFFOLDER block.
            FDR_ABRLEN_OFFSET       = 38,
            FDR_ABRLEN_LEN          = U1,
            // The offset of the field containing the
            // length of the abReserve field in each
            // CFDATA block.
            DATA_ABRLEN_OFFSET      = 39,
            DATA_ABRLEN_LEN         = U1
            ;

        /// <summary>
        /// Represents the flags within the
        /// cabinet file's header.
        /// </summary>
        [Flags]
        private enum CabinetHeaderFlags : ushort
        {
            /// <summary>
            /// Indicates that this cabinet file is not
            /// the first in a set of cabinet files.
            /// </summary>
            IsNotFirst      = 0x0001,
            /// <summary>
            /// Indicates that this cabinet file is not
            /// the last in a set of cabinet files.
            /// </summary>
            IsNotLast       = 0x0002,
            /// <summary>
            /// Indicates that the cabinet file contains
            /// the reserved fields cbCFHeader, cbCFFolder,
            /// and cbCFData in the current CFHEADER block.
            /// </summary>
            ReservedPresent = 0x0004
        }

        static CabinetArchive()
        {
            Signature = new byte[4] { 0x4d, 0x53, 0x43, 0x46 };
        }

        // The absolute offset of the first CFFILE block.
        private readonly uint _filesFirstOffset;
        private readonly ushort 
            // The number of files/folders stored within
            // the archive.
            _filesCount, _foldersCount,
            // The length of the CFHEADER block's abReserve
            // field.
            _headerAbLength;
        // The lengths of the abReserve fields in CFFOLDER
        // and CFDATA blocks, respectively.
        private readonly byte _folderAbLength, _dataAbLength;
        // Flags indicating the presence of optional fields
        // within the cabinet's main header.
        private readonly CabinetHeaderFlags _flags;

        // The names of the files stored within the archive.
        private List<string> _names;
        // A set of Filemark structs indicating the positions
        // of each file within the temporary file storing
        // their uncompressed representations.
        private List<Filemark> _marks;
        // A FileStream with the temporary file open and
        // ready for reading/writing.
        private FileStream _dataDump;
        // Object to use in locks when reading/writing
        // from the _dataDump FileStream.
        private object _dataLock;

        /// <summary>
        /// Writes a set of bytes to the dump file.
        /// </summary>
        /// <param name="name">The name of the file being written.</param>
        /// <param name="bytes">The contents of the file.</param>
        /// <param name="seekToEnd">Whether to seek to the end of the dump.</param>
        private void _writeToDump(string name, byte[] bytes, bool seekToEnd = true)
        {
            lock (_dataLock)
            {
                if (seekToEnd) _dataDump.Seek(0, SeekOrigin.End);

                _names.Add(name);
                _marks.Add(new Filemark(bytes.LongLength, _dataDump.Position));
                _dataDump.Write(bytes, 0, bytes.Length);
            }
        }

        /// <summary>
        /// Creates a new CabinetArchive.
        /// </summary>
        /// <param name="stream">The stream containing the archive's bytes.</param>
        /// <exception cref="System.ArgumentNullException">
        ///     Thrown when the provided stream is null.
        /// </exception>
        /// <exception cref="System.ArgumentException">
        ///     Thrown when the provided stream does not support the
        ///     required operations.
        /// </exception>
        /// <exception cref="System.IO.InvalidDataException">
        ///     Thrown when the stream's data does not pass verification.
        /// </exception>
        /// <exception cref="System.NotSupportedException">
        ///     Thrown when the file format version of the provided cabinet
        ///     file is not supported by the class.
        /// </exception>
        public CabinetArchive(Stream stream)
        {
            /* * * * * * * * * *
             * Set up the class to make it ready for cabinet parsing
             * * * * * * * * * */
            if (stream == null)
                throw new ArgumentNullException(
                    "The provided stream cannot be null.",
                    "stream"
                    );

            if (!stream.CanSeek)
                throw new ArgumentException(
                    "The provided stream does not support seeking."
                    );

            if (!stream.CanRead)
                throw new ArgumentException(
                    "The provided stream does not support reading."
                    );

            _names = new List<string>();
            _marks = new List<Filemark>();
            _dataLock = new object();

            try
            {
                _dataDump = new FileStream(
                    Path.GetTempFileName(),
                    FileMode.OpenOrCreate,
                    FileAccess.ReadWrite,
                    FileShare.None,
                    4096, // The default buffer size
                    FileOptions.DeleteOnClose | FileOptions.RandomAccess
                    );
            }
            catch (IOException ioex)
            {
                throw new IOException(
                    "Could not open a temporary file.",
                    ioex
                    );
            }
            catch (UnauthorizedAccessException uaex)
            {
                throw new IOException(
                    "Could not open a temporary file.",
                    uaex
                    );
            }

            /* * * * * * * * * *
             * Parse the cabinet file
             * * * * * * * * * */
            stream.Position = 0;

            // We know that these headers will be present
            // in every cabinet file, so we can just read
            // them.
            byte[] hdrBuf = new byte[HEADER_MIN_LENGTH];
            stream.Read(hdrBuf, 0, HEADER_MIN_LENGTH);
{'fraction_non_alphanumeric': 0.07849497242945183, 'fraction_numerical': 0.012195913071683426, 'mean_word_length': 1.2737463126843658, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '892256', 'n_tokens_mistral': 4041, 'n_tokens_neox': 3754, 'n_words': 1215}