<commit_before>#include "find_datarefs_in_files.h"

#include "logging.h"

#include <algorithm>
#include <cctype>
#include <fstream>
#include <sstream>
#include <string>
#include <unordered_set>

#include <boost/filesystem.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/algorithm/string.hpp>

// + character is necessary for A320neo
inline char isValidDatarefChar(char c) {
	return std::isalnum((unsigned char)(c)) || '_' == c || '/' == c || '-' == c || '.' == c || '+' == c;
}

const size_t min_dataref_length = 8;

std::vector<std::string> getDatarefsFromFile(const boost::filesystem::path & filename) {
    boost::iostreams::mapped_file_source file;
    
    try {
        file.open(filename);
    } catch(std::exception &) {
		return {};
    }

	std::vector<std::string> all_refs;
	size_t start_pos = 0;
	const char * file_data = file.data();
	bool last_char_valid = false;
    for(size_t i = 0; i < file.size(); i++) {
    	if(isValidDatarefChar(file_data[i])) {<commit_msg>Open more files in binary mode.<commit_after>#include "find_datarefs_in_files.h"

#include "logging.h"

#include <algorithm>
#include <cctype>
#include <fstream>
#include <sstream>
#include <string>
#include <unordered_set>

#include <boost/filesystem.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/algorithm/string.hpp>

// + character is necessary for A320neo
inline char isValidDatarefChar(char c) {
	return std::isalnum((unsigned char)(c)) || '_' == c || '/' == c || '-' == c || '.' == c || '+' == c;
}

const size_t min_dataref_length = 8;

std::vector<std::string> getDatarefsFromFile(const boost::filesystem::path & filename) {
    boost::iostreams::mapped_file_source file;
    
    try {
        file.open(filename, std::ios::in | std::ios::binary);
    } catch(std::exception &) {
		return {};
    }

	std::vector<std::string> all_refs;
	size_t start_pos = 0;
	const char * file_data = file.data();
	bool last_char_valid = false;
    for(size_t i = 0; i < file.size(); i++) {
    	if(isValidDatarefChar(file_data[i])) {
{'fraction_non_alphanumeric': 0.1708029197080292, 'fraction_numerical': 0.00583941605839416, 'mean_word_length': 4.617486338797814, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '496604', 'n_tokens_mistral': 764, 'n_tokens_neox': 747, 'n_words': 151}