<commit_before>#include "GameData.h"

#include <string>
#include <sstream>
#include <algorithm>

#include <map>
#include <zlib/zlib.h>

#include "bparse.h"
#include "DatCat.h"
#include "File.h"

namespace
{
    // Relation between category number and category name
    // These names are taken straight from the exe, it helps resolve dispatching when getting files by path

    std::unordered_map< std::string, uint32_t > categoryNameToIdMap = 
       {{"common",          0x00},
        {"bgcommon",        0x01},
        {"bg",              0x02},
        {"cut",             0x03},
        {"chara",           0x04},
        {"shader",          0x05},
        {"ui",              0x06},
        {"sound",           0x07},
        {"vfx",             0x08},
        {"ui_script",       0x09},
        {"exd",             0x0A},
        {"game_script",     0x0B},
        {"music",           0x0C}
   };

    std::unordered_map< uint32_t, std::string > categoryIdToNameMap = 
       {{0x00,              "common"},
        {0x01,              "bgcommon"},
        {0x02,              "bg"},
        {0x03,              "cut"},
        {0x04,              "chara"},
        {0x05,              "shader"},
        {0x06,              "ui"},
        {0x07,              "sound"},
        {0x08,              "vfx"},
        {0x09,              "ui_script"},
        {0x0A,              "exd"},
        {0x0B,              "game_script"},
        {0x0C,              "music"}};
}

namespace xiv
{
namespace dat
{

GameData::GameData(const std::experimental::filesystem::path& path) try :
    m_path(path)
{
   int maxExLevel = 0;
  
   // Determine which expansions are available
   while( std::experimental::filesystem::exists( std::experimental::filesystem::path( m_path.string() + "\\ex" + std::to_string( maxExLevel + 1) + "\\ex" + std::to_string( maxExLevel + 1) + ".ver" ) ) )
   {
      maxExLevel++;
   }
   

   // Iterate over the files in path
   for( auto it = std::experimental::filesystem::directory_iterator( m_path.string() + "//ffxiv" ); it != std::experimental::filesystem::directory_iterator(); ++it )
   {
      // Get the filename of the current element
      auto filename = it->path().filename().string();

      // If it contains ".win32.index" this is most likely a hit for a category
      if( filename.find( ".win32.index" ) != std::string::npos &&  filename.find( ".win32.index2" ) == std::string::npos )
      {
         // Format of indexes is XX0000.win32.index, so fetch the hex number for category number
         std::istringstream iss( filename.substr( 0, 2 ) );
         uint32_t cat_nb;
         iss >> std::hex >> cat_nb;


         // Add to the list of category number
         // creates the empty category in the cats map
         // instantiate the creation mutex for this category
         m_catNums.push_back( cat_nb );
         m_cats[cat_nb] = std::unique_ptr<Cat>();
         m_catCreationMutexes[cat_nb] = std::unique_ptr<std::mutex>( new std::mutex() );

         // Check for expansion
         for( int exNum = 1; exNum <= maxExLevel; exNum++ )
         {
            const std::string path = m_path.string() + "\\" + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, 0, "win32", "index" );

            if( std::experimental::filesystem::exists( std::experimental::filesystem::path( path ) ) )
            {

               int chunkCount = 0;

               for(int chunkTest = 0; chunkTest < 256; chunkTest++ )
               {
                  if( std::experimental::filesystem::exists( m_path.string() + "\\" + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, chunkTest, "win32", "index" ) ) )
                  {
                     m_exCats[cat_nb].exNumToChunkMap[exNum].chunkToCatMap[chunkTest] = std::unique_ptr<Cat>();
                     chunkCount++;
                  }
               }      

            }
         }
      }
   }

}
catch( std::exception& e )
{
   // In case of failure here, client is supposed to catch the exception because it is not recoverable on our side
   throw std::runtime_error( "GameData initialization failed: " + std::string( e.what() ) );
}

GameData::~GameData()
{

}

const std::string GameData::buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk, const std::string platform, const std::string type )
{
   char dat[1024];
   sprintf( dat, "%s/%02x%02x%02x.%s.%s", folder.c_str(), cat, exNum, chunk, platform.c_str(), type.c_str() );
   return std::string( dat );
}
<commit_msg>fix expansion data loading on linux<commit_after>#include "GameData.h"

#include <string>
#include <sstream>
#include <algorithm>

#include <map>
#include <zlib/zlib.h>

#include "bparse.h"
#include "DatCat.h"
#include "File.h"

namespace
{
    // Relation between category number and category name
    // These names are taken straight from the exe, it helps resolve dispatching when getting files by path

    std::unordered_map< std::string, uint32_t > categoryNameToIdMap = 
       {{"common",          0x00},
        {"bgcommon",        0x01},
        {"bg",              0x02},
        {"cut",             0x03},
        {"chara",           0x04},
        {"shader",          0x05},
        {"ui",              0x06},
        {"sound",           0x07},
        {"vfx",             0x08},
        {"ui_script",       0x09},
        {"exd",             0x0A},
        {"game_script",     0x0B},
        {"music",           0x0C}
   };

    std::unordered_map< uint32_t, std::string > categoryIdToNameMap = 
       {{0x00,              "common"},
        {0x01,              "bgcommon"},
        {0x02,              "bg"},
        {0x03,              "cut"},
        {0x04,              "chara"},
        {0x05,              "shader"},
        {0x06,              "ui"},
        {0x07,              "sound"},
        {0x08,              "vfx"},
        {0x09,              "ui_script"},
        {0x0A,              "exd"},
        {0x0B,              "game_script"},
        {0x0C,              "music"}};
}

namespace xiv
{
namespace dat
{

GameData::GameData(const std::experimental::filesystem::path& path) try :
    m_path(path)
{
   int maxExLevel = 0;

   auto sep = std::experimental::filesystem::path::preferred_separator;
  
   // Determine which expansions are available
   while( std::experimental::filesystem::exists( std::experimental::filesystem::path( m_path.string() + sep + "ex" + std::to_string( maxExLevel + 1 ) + sep + "ex" + std::to_string( maxExLevel + 1 ) + ".ver" ) ) )
   {
      maxExLevel++;
   }
   

   // Iterate over the files in path
   for( auto it = std::experimental::filesystem::directory_iterator( m_path.string() + "//ffxiv" ); it != std::experimental::filesystem::directory_iterator(); ++it )
   {
      // Get the filename of the current element
      auto filename = it->path().filename().string();

      // If it contains ".win32.index" this is most likely a hit for a category
      if( filename.find( ".win32.index" ) != std::string::npos &&  filename.find( ".win32.index2" ) == std::string::npos )
      {
         // Format of indexes is XX0000.win32.index, so fetch the hex number for category number
         std::istringstream iss( filename.substr( 0, 2 ) );
         uint32_t cat_nb;
         iss >> std::hex >> cat_nb;


         // Add to the list of category number
         // creates the empty category in the cats map
         // instantiate the creation mutex for this category
         m_catNums.push_back( cat_nb );
         m_cats[cat_nb] = std::unique_ptr<Cat>();
         m_catCreationMutexes[cat_nb] = std::unique_ptr<std::mutex>( new std::mutex() );

         // Check for expansion
         for( int exNum = 1; exNum <= maxExLevel; exNum++ )
         {
            const std::string path = m_path.string() + sep + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, 0, "win32", "index" );

            if( std::experimental::filesystem::exists( std::experimental::filesystem::path( path ) ) )
            {

               int chunkCount = 0;

               for(int chunkTest = 0; chunkTest < 256; chunkTest++ )
               {
                  if( std::experimental::filesystem::exists( m_path.string() + sep + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, chunkTest, "win32", "index" ) ) )
                  {
                     m_exCats[cat_nb].exNumToChunkMap[exNum].chunkToCatMap[chunkTest] = std::unique_ptr<Cat>();
                     chunkCount++;
                  }
               }      

            }
         }
      }
   }

}
catch( std::exception& e )
{
   // In case of failure here, client is supposed to catch the exception because it is not recoverable on our side
   throw std::runtime_error( "GameData initialization failed: " + std::string( e.what() ) );
}

GameData::~GameData()
{

}

const std::string GameData::buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk, const std::string platform, const std::string type )
{
   char dat[1024];
   sprintf( dat, "%s/%02x%02x%02x.%s.%s", folder.c_str(), cat, exNum, chunk, platform.c_str(), type.c_str() );
   return std::string( dat );
}

{'fraction_non_alphanumeric': 0.14474114441416894, 'fraction_numerical': 0.02550408719346049, 'mean_word_length': 2.0404241219350565, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7191437', 'n_tokens_mistral': 2987, 'n_tokens_neox': 2816, 'n_words': 710}