<commit_before>
#include <GameData.h>
#include <File.h>
#include <DatCat.h>
#include <ExdData.h>
#include <ExdCat.h>
#include <Exd.h>
#include <Exh.h>
#include <iostream>
#include <cctype>
#include <set>
#include <Exd/ExdDataGenerated.h>
#include <Logging/Logger.h>

#include <fstream>
#include <streambuf>
#include <regex>

#include <datReader/DatCategories/bg/pcb.h>
#include <datReader/DatCategories/bg/lgb.h>
#include <datReader/DatCategories/bg/sgb.h>

#include <Util/CrashHandler.h>

#include <experimental/filesystem>

Sapphire::Common::Util::CrashHandler crashHandler;

Sapphire::Data::ExdDataGenerated g_exdData;
xiv::dat::GameData* gameData = nullptr;

using namespace Sapphire;

namespace fs = std::experimental::filesystem;

//const std::string datLocation( "/opt/sapphire_3_15_0/bin/sqpack" );
const std::string datLocation( "/mnt/c/Program Files (x86)/Steam/steamapps/common/FINAL FANTASY XIV Online/game/sqpack" );

void exportFile( const std::string& path )
{
  try
  {
    auto file = gameData->getFile( path );

    if( !file )
    {
      return;
    }
    auto p = fs::path( path );
    fs::create_directories( p.parent_path() );
    file->exportToFile( p );
  }
  catch( const std::exception& ex )
  {
    Logger::error( "Failed to export file {0}, error: {1}", path, ex.what() );
  }
}

const std::string getTypeString( uint32_t type )
{
  switch( type )
  {
    case 1: return "BgParts";
    case 3: return "Light";
    case 4: return "Vfx";
    case 5: return "PositionMarker";
    case 6: return "Gimmick/SharedGroup6";
    case 7: return "Sound";
    case 8: return "EventNpc";
    case 9: return "BattleNpc";
    case 12: return "Aetheryte";
    case 13: return "EnvSpace";
    case 14: return "Gathering";
    case 15: return "SharedGroup15";
    case 16: return "Treasure";
    case 39: return "Weapon";
    case 40: return "PopRange";
    case 41: return "ExitRange";
    case 43: return "MapRange";
    case 44: return "NaviMeshRange";
    case 45: return "EventObject";
    case 47: return "EnvLocation";
    case 49: return "EventRange";
    case 51: return "QuestMarker";
    case 57: return "CollisionBox";
    case 58: return "DoorRange";
    case 59: return "LineVfx";
    case 65: return "ClientPath";
    case 66: return "ServerPath";
    case 67: return "GimmickRange";
    case 68: return "TargetMarker";
    case 69: return "ChairMarker";
    case 70: return "ClickableRange";
    case 71: return "PrefetchRange";
    case 72: return "FateRange";
    case 75: return "SphereCastRange";

    default: return "";
  }
}

struct DupeResult
{
  std::string groupName;
  std::string lgb;
  uint32_t id;
};

int main( int argc, char* argv[] )
{

  Logger::init( "struct_test" );

  assert(false);

  Logger::info( "Setting up EXD data" );
  if( !g_exdData.init( datLocation ) )
  {
    Logger::fatal( "Error setting up EXD data " );
    return 0;
  }

  gameData = new xiv::dat::GameData( datLocation );

  if( argc > 1 )
  {
    while( argc-- > 1 )
    {
      Logger::info( "Exporting file: {}", argv[ argc ] );
      exportFile( argv[ argc ] );
    }

    return 0;
  }<commit_msg>remove assert left in for testing<commit_after>
#include <GameData.h>
#include <File.h>
#include <DatCat.h>
#include <ExdData.h>
#include <ExdCat.h>
#include <Exd.h>
#include <Exh.h>
#include <iostream>
#include <cctype>
#include <set>
#include <Exd/ExdDataGenerated.h>
#include <Logging/Logger.h>

#include <fstream>
#include <streambuf>
#include <regex>

#include <datReader/DatCategories/bg/pcb.h>
#include <datReader/DatCategories/bg/lgb.h>
#include <datReader/DatCategories/bg/sgb.h>

#include <Util/CrashHandler.h>

#include <experimental/filesystem>

Sapphire::Common::Util::CrashHandler crashHandler;

Sapphire::Data::ExdDataGenerated g_exdData;
xiv::dat::GameData* gameData = nullptr;

using namespace Sapphire;

namespace fs = std::experimental::filesystem;

//const std::string datLocation( "/opt/sapphire_3_15_0/bin/sqpack" );
const std::string datLocation( "/mnt/c/Program Files (x86)/Steam/steamapps/common/FINAL FANTASY XIV Online/game/sqpack" );

void exportFile( const std::string& path )
{
  try
  {
    auto file = gameData->getFile( path );

    if( !file )
    {
      return;
    }
    auto p = fs::path( path );
    fs::create_directories( p.parent_path() );
    file->exportToFile( p );
  }
  catch( const std::exception& ex )
  {
    Logger::error( "Failed to export file {0}, error: {1}", path, ex.what() );
  }
}

const std::string getTypeString( uint32_t type )
{
  switch( type )
  {
    case 1: return "BgParts";
    case 3: return "Light";
    case 4: return "Vfx";
    case 5: return "PositionMarker";
    case 6: return "Gimmick/SharedGroup6";
    case 7: return "Sound";
    case 8: return "EventNpc";
    case 9: return "BattleNpc";
    case 12: return "Aetheryte";
    case 13: return "EnvSpace";
    case 14: return "Gathering";
    case 15: return "SharedGroup15";
    case 16: return "Treasure";
    case 39: return "Weapon";
    case 40: return "PopRange";
    case 41: return "ExitRange";
    case 43: return "MapRange";
    case 44: return "NaviMeshRange";
    case 45: return "EventObject";
    case 47: return "EnvLocation";
    case 49: return "EventRange";
    case 51: return "QuestMarker";
    case 57: return "CollisionBox";
    case 58: return "DoorRange";
    case 59: return "LineVfx";
    case 65: return "ClientPath";
    case 66: return "ServerPath";
    case 67: return "GimmickRange";
    case 68: return "TargetMarker";
    case 69: return "ChairMarker";
    case 70: return "ClickableRange";
    case 71: return "PrefetchRange";
    case 72: return "FateRange";
    case 75: return "SphereCastRange";

    default: return "";
  }
}

struct DupeResult
{
  std::string groupName;
  std::string lgb;
  uint32_t id;
};

int main( int argc, char* argv[] )
{

  Logger::init( "struct_test" );

  Logger::info( "Setting up EXD data" );
  if( !g_exdData.init( datLocation ) )
  {
    Logger::fatal( "Error setting up EXD data " );
    return 0;
  }

  gameData = new xiv::dat::GameData( datLocation );

  if( argc > 1 )
  {
    while( argc-- > 1 )
    {
      Logger::info( "Exporting file: {}", argv[ argc ] );
      exportFile( argv[ argc ] );
    }

    return 0;
  }
{'fraction_non_alphanumeric': 0.1440705128205128, 'fraction_numerical': 0.02532051282051282, 'mean_word_length': 3.5621345029239766, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7039713', 'n_tokens_mistral': 2361, 'n_tokens_neox': 2186, 'n_words': 604}