<commit_before>#include "mfilescanner.h"

#include <cassert>
#include <cstring>
#include <cstdlib>
#include <iostream>
#include <algorithm>
#include <iterator>
#ifndef WIN32
#include <linux/limits.h>
#else
#define PATH_MAX 4096
#endif

using std::cerr;
using std::cout;
using std::cin;
using std::endl;
using std::string;
using std::vector;
using std::list;
using std::copy;
using std::map;
using std::set;
using std::istream;
using std::ifstream;<commit_msg>added suggestion for mac path to limits.h from georg ogris<commit_after>#include "mfilescanner.h"

#include <cassert>
#include <cstring>
#include <cstdlib>
#include <iostream>
#include <algorithm>
#include <iterator>
#ifdef WIN32
#define PATH_MAX 4096
#else
#ifdef __APPLE__
#include <limits.h>
#else
#include <linux/limits.h>
#endif
#endif

using std::cerr;
using std::cout;
using std::cin;
using std::endl;
using std::string;
using std::vector;
using std::list;
using std::copy;
using std::map;
using std::set;
using std::istream;
using std::ifstream;
{'fraction_non_alphanumeric': 0.14624505928853754, 'fraction_numerical': 0.011857707509881422, 'mean_word_length': 7.441666666666666, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2522386', 'n_tokens_mistral': 376, 'n_tokens_neox': 362, 'n_words': 108}