<commit_before>#include "framework/data.h"
#include "framework/framework.h"
#include "game/state/battle/battle.h"
#include "game/state/battle/battlemap.h"
#include "tools/extractors/common/battlemap.h"
#include "tools/extractors/extractors.h"

#include <map>

namespace OpenApoc
{

void InitialGameStateExtractor::extractBattlescapeMapFromPath(GameState &state,
                                                              const UString dirName,
                                                              const int index)
{
	UString tilePrefix = UString::format("%s_", dirName);
	UString map_prefix = "xcom3/maps/";
	UString mapunits_suffix = "/mapunits/";

	BuildingDatStructure bdata;
	{
		auto fileName = dirName + UString("/building.dat");

		auto datFileName = map_prefix + fileName;
		auto inFile = fw().data->fs.open(datFileName);
		if (!inFile)
		{
			LogError("Failed to open \"%s\"", fileName.cStr());
			return;
		}

		inFile.read((char *)&bdata, sizeof(bdata));
		if (!inFile)
		{
			LogError("Failed to read entry in \"%s\"", fileName.cStr());
			return;
		}
	}

	RubbleDatStructure rdata;
	{
		auto fileName = dirName + mapunits_suffix + UString("rubble.dat");
		auto fullPath = map_prefix + fileName;
		auto inFile = fw().data->fs.open(fullPath);
		if (!inFile)
		{
			LogError("Failed to open \"%s\"", fileName.cStr());
			return;
		}

		inFile.read((char *)&rdata, sizeof(rdata));
		if (!inFile)
		{
			LogError("Failed to read entry in \"%s\"", fileName.cStr());
			return;
		}
	}

	int firstExitIdx = 0;
	{
		auto fileName = dirName + mapunits_suffix + UString("grounmap.dat");

		auto fullPath = map_prefix + fileName;
		auto inFile = fw().data->fs.open(fullPath);
		if (!inFile)
		{
			LogError("Failed to open \"%s\"", fileName.cStr());
			return;
		}

		auto fileSize = inFile.size();
		auto objectCount = fileSize / sizeof(struct BattleMapPartEntry);
		firstExitIdx = objectCount - 4;
	}

	auto m = mksp<BattleMap>();

	UString id = UString::format("%s%s", BattleMap::getPrefix(), this->battleMapPaths[index]);

	m->id = id;
	m->chunk_size = {bdata.chunk_x, bdata.chunk_y, bdata.chunk_z};
	m->max_battle_size = {bdata.battle_x, bdata.battle_y, bdata.battle_z};

	uint8_t north_flag = 0b0001;
	uint8_t east_flag = 0b0010;
	uint8_t south_flag = 0b0100;
	uint8_t west_flag = 0b1000;

	m->allow_entrance[Battle::MapBorder::North] = bdata.allow_entrance_from & north_flag;
	m->allow_entrance[Battle::MapBorder::East] = bdata.allow_entrance_from & east_flag;
	m->allow_entrance[Battle::MapBorder::South] = bdata.allow_entrance_from & south_flag;
	m->allow_entrance[Battle::MapBorder::West] = bdata.allow_entrance_from & west_flag;
	m->allow_exit[Battle::MapBorder::North] = bdata.allow_exit_from & north_flag;
	m->allow_exit[Battle::MapBorder::East] = bdata.allow_exit_from & east_flag;
	m->allow_exit[Battle::MapBorder::South] = bdata.allow_exit_from & south_flag;
	m->allow_exit[Battle::MapBorder::West] = bdata.allow_exit_from & west_flag;
	m->entrance_level_min = bdata.entrance_min_level;
	m->entrance_level_max = bdata.entrance_max_level;
	m->exit_level_min = bdata.exit_min_level;
	m->exit_level_max = bdata.exit_max_level;
	m->tilesets.emplace_back(tilePrefix.substr(0, tilePrefix.length() - 1));

	// Side 0 = exits by X axis, Side 1 = exits by Y axis
	for (int l = 0; l < 15; l++)
	{
		for (int e = 0; e < 14; e++)
		{
			if (bdata.exits[0][l].exits[e] != 0xffffffff)
				m->exitsX.emplace(bdata.exits[0][l].exits[e], 0, l);
			if (bdata.exits[1][l].exits[e] != 0xffffffff)
				m->exitsY.emplace(0, bdata.exits[1][l].exits[e], l);
		}
	}

	if (bdata.destroyed_ground_idx != 0)
		m->destroyed_ground_tile = {
		    &state, UString::format("%s%s%s%u", BattleMapPartType::getPrefix(), tilePrefix, "GD_",
		                            (unsigned)bdata.destroyed_ground_idx)};

	for (int i = 0; i < 5; i++)
	{
		if (rdata.left_wall[i] != 0)
		{
			m->rubble_left_wall.emplace_back(
			    &state, UString::format("%s%s%s%u", BattleMapPartType::getPrefix(), tilePrefix,
			                            "LW_", (unsigned)rdata.left_wall[i]));
		}
		if (rdata.right_wall[i] != 0)
		{
			m->rubble_right_wall.emplace_back(
			    &state, UString::format("%s%s%s%u", BattleMapPartType::getPrefix(), tilePrefix,
			                            "RW_", (unsigned)rdata.right_wall[i]));
		}
		if (rdata.feature[i] != 0)
		{
			m->rubble_feature.emplace_back(<commit_msg>Fix misaligned pointer access in extract_battlescape_map<commit_after>#include "framework/data.h"
#include "framework/framework.h"
#include "game/state/battle/battle.h"
#include "game/state/battle/battlemap.h"
#include "tools/extractors/common/battlemap.h"
#include "tools/extractors/extractors.h"

#include <map>

namespace OpenApoc
{

void InitialGameStateExtractor::extractBattlescapeMapFromPath(GameState &state,
                                                              const UString dirName,
                                                              const int index)
{
	UString tilePrefix = UString::format("%s_", dirName);
	UString map_prefix = "xcom3/maps/";
	UString mapunits_suffix = "/mapunits/";

	BuildingDatStructure bdata;
	{
		auto fileName = dirName + UString("/building.dat");

		auto datFileName = map_prefix + fileName;
		auto inFile = fw().data->fs.open(datFileName);
		if (!inFile)
		{
			LogError("Failed to open \"%s\"", fileName.cStr());
			return;
		}

		inFile.read((char *)&bdata, sizeof(bdata));
		if (!inFile)
		{
			LogError("Failed to read entry in \"%s\"", fileName.cStr());
			return;
		}
	}

	RubbleDatStructure rdata;
	{
		auto fileName = dirName + mapunits_suffix + UString("rubble.dat");
		auto fullPath = map_prefix + fileName;
		auto inFile = fw().data->fs.open(fullPath);
		if (!inFile)
		{
			LogError("Failed to open \"%s\"", fileName.cStr());
			return;
		}

		inFile.read((char *)&rdata, sizeof(rdata));
		if (!inFile)
		{
			LogError("Failed to read entry in \"%s\"", fileName.cStr());
			return;
		}
	}

	int firstExitIdx = 0;
	{
		auto fileName = dirName + mapunits_suffix + UString("grounmap.dat");

		auto fullPath = map_prefix + fileName;
		auto inFile = fw().data->fs.open(fullPath);
		if (!inFile)
		{
			LogError("Failed to open \"%s\"", fileName.cStr());
			return;
		}

		auto fileSize = inFile.size();
		auto objectCount = fileSize / sizeof(struct BattleMapPartEntry);
		firstExitIdx = objectCount - 4;
	}

	auto m = mksp<BattleMap>();

	UString id = UString::format("%s%s", BattleMap::getPrefix(), this->battleMapPaths[index]);

	m->id = id;
	m->chunk_size = {bdata.chunk_x, bdata.chunk_y, bdata.chunk_z};
	m->max_battle_size = {bdata.battle_x, bdata.battle_y, bdata.battle_z};

	uint8_t north_flag = 0b0001;
	uint8_t east_flag = 0b0010;
	uint8_t south_flag = 0b0100;
	uint8_t west_flag = 0b1000;

	m->allow_entrance[Battle::MapBorder::North] = bdata.allow_entrance_from & north_flag;
	m->allow_entrance[Battle::MapBorder::East] = bdata.allow_entrance_from & east_flag;
	m->allow_entrance[Battle::MapBorder::South] = bdata.allow_entrance_from & south_flag;
	m->allow_entrance[Battle::MapBorder::West] = bdata.allow_entrance_from & west_flag;
	m->allow_exit[Battle::MapBorder::North] = bdata.allow_exit_from & north_flag;
	m->allow_exit[Battle::MapBorder::East] = bdata.allow_exit_from & east_flag;
	m->allow_exit[Battle::MapBorder::South] = bdata.allow_exit_from & south_flag;
	m->allow_exit[Battle::MapBorder::West] = bdata.allow_exit_from & west_flag;
	m->entrance_level_min = bdata.entrance_min_level;
	m->entrance_level_max = bdata.entrance_max_level;
	m->exit_level_min = bdata.exit_min_level;
	m->exit_level_max = bdata.exit_max_level;
	m->tilesets.emplace_back(tilePrefix.substr(0, tilePrefix.length() - 1));

	// Side 0 = exits by X axis, Side 1 = exits by Y axis
	for (int l = 0; l < 15; l++)
	{
		for (int e = 0; e < 14; e++)
		{
			/* As the exits array isn't uint32_t aligned, pull it out rather than passing a
			 * reference directly to the Vec3<> constructor
			 * Copying it out to the stack here allows the compiler to do whatever lowering is
			 * necessary to realign the reads*/
			uint32_t val = bdata.exits[0][l].exits[e];
			if (val != 0xffffffff)
				m->exitsX.emplace(val, 0, l);
			val = bdata.exits[1][l].exits[e];
			if (val != 0xffffffff)
				m->exitsY.emplace(0, val, l);
		}
	}

	if (bdata.destroyed_ground_idx != 0)
		m->destroyed_ground_tile = {
		    &state, UString::format("%s%s%s%u", BattleMapPartType::getPrefix(), tilePrefix, "GD_",
		                            (unsigned)bdata.destroyed_ground_idx)};

	for (int i = 0; i < 5; i++)
	{
		if (rdata.left_wall[i] != 0)
		{
			m->rubble_left_wall.emplace_back(
			    &state, UString::format("%s%s%s%u", BattleMapPartType::getPrefix(), tilePrefix,
			                            "LW_", (unsigned)rdata.left_wall[i]));
		}
		if (rdata.right_wall[i] != 0)
		{
			m->rubble_right_wall.emplace_back(
			    &state, UString::format("%s%s%s%u", BattleMapPartType::getPrefix(), tilePrefix,
			                            "RW_", (unsigned)rdata.right_wall[i]));
		}
		if (rdata.feature[i] != 0)
		{
			m->rubble_feature.emplace_back(
{'fraction_non_alphanumeric': 0.15373579075157268, 'fraction_numerical': 0.011588124931023066, 'mean_word_length': 3.6519507186858315, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1360188', 'n_tokens_mistral': 3644, 'n_tokens_neox': 3346, 'n_words': 613}