<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using ICSharpCode.SharpZipLib.Zip;

namespace SourceUtils
{
    partial class ValveBspFile
    {
        public class PakFileLump : ILump, IResourceProvider, IDisposable
        {
            /// <summary>
            /// If true, will write to a file to help debug the contents.
            /// </summary>
            public static bool DebugContents { get; set; }

            [ThreadStatic]
            private static Dictionary<PakFileLump, ZipFile> _sArchivePool;

            private static ZipFile GetZipArchive( PakFileLump pak )
            {
                if (_sArchivePool == null) _sArchivePool = new Dictionary<PakFileLump, ZipFile>();

                ZipFile archive;
                if ( _sArchivePool.TryGetValue( pak, out archive ) ) return archive;

                archive = new ZipFile( pak._bspFile.GetLumpStream( pak.LumpType ) );
                _sArchivePool.Add( pak, archive );

                lock ( pak._threadArchives )
                {
                    pak._threadArchives.Add( archive );
                }

                return archive;
            }

            public LumpType LumpType { get; }

            private readonly List<ZipFile> _threadArchives = new List<ZipFile>();

            private readonly ValveBspFile _bspFile;
            private bool _loaded;

            private readonly Dictionary<string, int> _entryDict =
                new Dictionary<string, int>( StringComparer.InvariantCultureIgnoreCase );

            public PakFileLump( ValveBspFile bspFile, LumpType type )
            {
                _bspFile = bspFile;
                _loaded = false;
                LumpType = type;
            }

            private void EnsureLoaded()
            {
                lock ( this )
                {
                    if ( _loaded ) return;
                    _loaded = true;

                    if ( DebugContents )
                    {
                        using ( var stream = _bspFile.GetLumpStream( LumpType ) )
                        {
                            var bytes = new byte[stream.Length];
                            stream.Read( bytes, 0, bytes.Length );
                            File.WriteAllBytes( $"{_bspFile.Name}.pakfile.zip", bytes );
                        }
                    }

                    using ( var archive = new ZipFile( _bspFile.GetLumpStream( LumpType ) ) )
                    {
                        _entryDict.Clear();
                        for ( var i = 0; i < archive.Count; ++i )
                        {
                            var entry = archive[i];
                            var path = $"/{entry.Name}";
                            if ( !entry.IsFile || _entryDict.ContainsKey( path ) ) continue;
                            _entryDict.Add( path, i );
                        }
                    }
                }
            }

            public IEnumerable<string> GetFiles( string directory = "" )
            {
                var prefix = $"{directory}/";

                EnsureLoaded();<commit_msg>Fixed pakfile entries that contain backslashes<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using ICSharpCode.SharpZipLib.Zip;

namespace SourceUtils
{
    partial class ValveBspFile
    {
        public class PakFileLump : ILump, IResourceProvider, IDisposable
        {
            /// <summary>
            /// If true, will write to a file to help debug the contents.
            /// </summary>
            public static bool DebugContents { get; set; }

            [ThreadStatic]
            private static Dictionary<PakFileLump, ZipFile> _sArchivePool;

            private static ZipFile GetZipArchive( PakFileLump pak )
            {
                if (_sArchivePool == null) _sArchivePool = new Dictionary<PakFileLump, ZipFile>();

                ZipFile archive;
                if ( _sArchivePool.TryGetValue( pak, out archive ) ) return archive;

                archive = new ZipFile( pak._bspFile.GetLumpStream( pak.LumpType ) );
                _sArchivePool.Add( pak, archive );

                lock ( pak._threadArchives )
                {
                    pak._threadArchives.Add( archive );
                }

                return archive;
            }

            public LumpType LumpType { get; }

            private readonly List<ZipFile> _threadArchives = new List<ZipFile>();

            private readonly ValveBspFile _bspFile;
            private bool _loaded;

            private readonly Dictionary<string, int> _entryDict =
                new Dictionary<string, int>( StringComparer.InvariantCultureIgnoreCase );

            public PakFileLump( ValveBspFile bspFile, LumpType type )
            {
                _bspFile = bspFile;
                _loaded = false;
                LumpType = type;
            }

            private void EnsureLoaded()
            {
                lock ( this )
                {
                    if ( _loaded ) return;
                    _loaded = true;

                    if ( DebugContents )
                    {
                        using ( var stream = _bspFile.GetLumpStream( LumpType ) )
                        {
                            var bytes = new byte[stream.Length];
                            stream.Read( bytes, 0, bytes.Length );
                            File.WriteAllBytes( $"{_bspFile.Name}.pakfile.zip", bytes );
                        }
                    }

                    using ( var archive = new ZipFile( _bspFile.GetLumpStream( LumpType ) ) )
                    {
                        _entryDict.Clear();
                        for ( var i = 0; i < archive.Count; ++i )
                        {
                            var entry = archive[i];
                            var path = $"/{entry.Name.Replace( '\\', '/' )}";
                            if ( !entry.IsFile || _entryDict.ContainsKey( path ) ) continue;
                            _entryDict.Add( path, i );
                        }
                    }
                }
            }

            public IEnumerable<string> GetFiles( string directory = "" )
            {
                var prefix = $"{directory}/";

                EnsureLoaded();
{'fraction_non_alphanumeric': 0.08084772370486656, 'fraction_numerical': 0.0006279434850863422, 'mean_word_length': 1.1870923446618606, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2996538', 'n_tokens_mistral': 1657, 'n_tokens_neox': 1516, 'n_words': 378}