<commit_before>
#include "allrefs.h"
#include "logging.h"
#include "search.h"

#include <algorithm>
#include <cstdint>
#include <functional>
#include <fstream>
#include <locale>
#include <regex>
#include <sstream>
#include <unordered_set>
#include <vector>

#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/optional.hpp>

#include <signal.h>

void RefRecords::saveToFile(const boost::filesystem::path & dataref_filename, const boost::filesystem::path & commandref_filename) const {
	std::vector<const std::string *> dataref_names, commandref_names;
	dataref_names.reserve(datarefs.size());
	for(const DataRefRecord & dr : datarefs) {
		dataref_names.push_back(&dr.getName());
	}

	commandref_names.reserve(commandrefs.size());
	for(const CommandRefRecord & cr : commandrefs) {
		commandref_names.push_back(&cr.getName());
	}

	auto sort_and_write_names = [](std::vector<const std::string *> & names, const boost::filesystem::path & filename) -> bool {
		auto p_str_comparator = [](const std::string * s1, const std::string * s2) -> bool {
			return boost::ilexicographical_compare(*s1, *s2);
		};
		std::sort(names.begin(), names.end(), p_str_comparator);

		std::ofstream f(filename.string());
		for(const std::string * pstr : names) {
			f << *pstr << "\n";
		}
		f.close();
		if(f.fail()) {
			LOG("Error writing file " + filename.string());
		}
		return f.fail();
	};

	sort_and_write_names(dataref_names, dataref_filename);
	sort_and_write_names(commandref_names, commandref_filename);
}

std::vector<RefRecord *> RefRecords::add(const std::vector<std::string> & names, ref_src_t source) {
	std::string name;
	std::vector<RefRecord *> new_records;
	for(const std::string & name_untrimmed : names) {
		name = name_untrimmed;
		boost::algorithm::trim(name);

		//check for duplicates:
        NameMapType::iterator existing_location = ref_names_loaded.find(name);
        if(ref_names_loaded.cend() != existing_location) {
			continue;
		}

		XPLMDataRef dr = XPLMFindDataRef(name.c_str());
		XPLMCommandRef cr = XPLMFindCommand(name.c_str());
        if(nullptr != dr) {
        	datarefs.emplace_back(name, dr, source);
			new_records.emplace_back(&datarefs.back());
			dr_pointers.emplace_back(&datarefs.back());
			ref_names_loaded.insert(name);
		}

		if(nullptr != cr) {
        	commandrefs.emplace_back(name, cr, source);
			new_records.emplace_back(&commandrefs.back());
			cr_pointers.emplace_back(&commandrefs.back());
			ref_names_loaded.insert(name);
		}
	}

	return new_records;
}

const char * current_dr_name = nullptr;

void sig_handler(int) {
	XPLMDebugString("DRT: Crash detected while reading dataref ");
	XPLMDebugString(current_dr_name);
	XPLMDebugString("\n");
	XPLMDebugString("Please add this dataref to X-Plane 11/Resources/plugins/drt_blacklist.txt to prevent this happening again.\n");
	exit(-1);
}

std::vector<RefRecord *> RefRecords::update() {
	std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
	updater.updateTime(now);
	std::vector<RefRecord *> changed_drs;

#if IBM
	typedef SignalHandlerPointer sig_t;
#endif

	sig_t previous_sigsegv = signal(SIGSEGV, sig_handler);
	sig_t previous_sigfpe = signal(SIGFPE, sig_handler);

	for(DataRefRecord & dr : datarefs) {
        if(false == dr.isBlacklisted()) {
			current_dr_name = dr.getName().c_str();

            if(dr.update(updater)) { //if updated
				changed_drs.push_back(&dr);
			}
        }
	}

	signal(SIGSEGV, previous_sigsegv);
	signal(SIGFPE, previous_sigfpe);
	current_dr_name = nullptr;

	return changed_drs;
}
<commit_msg>One more try working around windows signal support<commit_after>
#include "allrefs.h"
#include "logging.h"
#include "search.h"

#include <algorithm>
#include <cstdint>
#include <functional>
#include <fstream>
#include <locale>
#include <regex>
#include <sstream>
#include <unordered_set>
#include <vector>

#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/optional.hpp>

#include <signal.h>

void RefRecords::saveToFile(const boost::filesystem::path & dataref_filename, const boost::filesystem::path & commandref_filename) const {
	std::vector<const std::string *> dataref_names, commandref_names;
	dataref_names.reserve(datarefs.size());
	for(const DataRefRecord & dr : datarefs) {
		dataref_names.push_back(&dr.getName());
	}

	commandref_names.reserve(commandrefs.size());
	for(const CommandRefRecord & cr : commandrefs) {
		commandref_names.push_back(&cr.getName());
	}

	auto sort_and_write_names = [](std::vector<const std::string *> & names, const boost::filesystem::path & filename) -> bool {
		auto p_str_comparator = [](const std::string * s1, const std::string * s2) -> bool {
			return boost::ilexicographical_compare(*s1, *s2);
		};
		std::sort(names.begin(), names.end(), p_str_comparator);

		std::ofstream f(filename.string());
		for(const std::string * pstr : names) {
			f << *pstr << "\n";
		}
		f.close();
		if(f.fail()) {
			LOG("Error writing file " + filename.string());
		}
		return f.fail();
	};

	sort_and_write_names(dataref_names, dataref_filename);
	sort_and_write_names(commandref_names, commandref_filename);
}

std::vector<RefRecord *> RefRecords::add(const std::vector<std::string> & names, ref_src_t source) {
	std::string name;
	std::vector<RefRecord *> new_records;
	for(const std::string & name_untrimmed : names) {
		name = name_untrimmed;
		boost::algorithm::trim(name);

		//check for duplicates:
        NameMapType::iterator existing_location = ref_names_loaded.find(name);
        if(ref_names_loaded.cend() != existing_location) {
			continue;
		}

		XPLMDataRef dr = XPLMFindDataRef(name.c_str());
		XPLMCommandRef cr = XPLMFindCommand(name.c_str());
        if(nullptr != dr) {
        	datarefs.emplace_back(name, dr, source);
			new_records.emplace_back(&datarefs.back());
			dr_pointers.emplace_back(&datarefs.back());
			ref_names_loaded.insert(name);
		}

		if(nullptr != cr) {
        	commandrefs.emplace_back(name, cr, source);
			new_records.emplace_back(&commandrefs.back());
			cr_pointers.emplace_back(&commandrefs.back());
			ref_names_loaded.insert(name);
		}
	}

	return new_records;
}

const char * current_dr_name = nullptr;

void sig_handler(int) {
	XPLMDebugString("DRT: Crash detected while reading dataref ");
	XPLMDebugString(current_dr_name);
	XPLMDebugString("\n");
	XPLMDebugString("Please add this dataref to X-Plane 11/Resources/plugins/drt_blacklist.txt to prevent this happening again.\n");
	exit(-1);
}

std::vector<RefRecord *> RefRecords::update() {
	std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
	updater.updateTime(now);
	std::vector<RefRecord *> changed_drs;

#if IBM
    typedef void (*sig_t)(int);
#endif

	sig_t previous_sigsegv = signal(SIGSEGV, sig_handler);
	sig_t previous_sigfpe = signal(SIGFPE, sig_handler);

	for(DataRefRecord & dr : datarefs) {
        if(false == dr.isBlacklisted()) {
			current_dr_name = dr.getName().c_str();

            if(dr.update(updater)) { //if updated
				changed_drs.push_back(&dr);
			}
        }
	}

	signal(SIGSEGV, previous_sigsegv);
	signal(SIGFPE, previous_sigfpe);
	current_dr_name = nullptr;

	return changed_drs;
}

{'fraction_non_alphanumeric': 0.14792008757525998, 'fraction_numerical': 0.0019157088122605363, 'mean_word_length': 5.241673783091375, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3047943', 'n_tokens_mistral': 2724, 'n_tokens_neox': 2594, 'n_words': 451}