<commit_before>#ifdef __cplusplus
extern "C" {
#endif
#define PERL_NO_GET_CONTEXT
#include "EXTERN.h"
#include "perl.h"
#include "XSUB.h"
#ifdef __cplusplus
}
#endif

#ifdef do_open
#undef do_open
#endif
#ifdef do_close
#undef do_close
#endif

#define NEED_sv_2pvbyte
#define NEED_newCONSTSUB
#define NEED_newRV_noinc
#define NEED_newSVpvn_flags
#define NEED_sv_2pv_flags
#include "ppport.h"

#include <sstream>
#include "ux.hpp"
<commit_msg>Hack to work around an error on Clang<commit_after>#ifdef __cplusplus
extern "C" {
#endif
#define PERL_NO_GET_CONTEXT
#include "EXTERN.h"
#include "perl.h"
#include "XSUB.h"
#ifdef __cplusplus
}
#endif

// Hack to work around "error: declaration of 'Perl___notused' has a different
// language linkage" error on Clang
#ifdef dNOOP
# undef dNOOP
# define dNOOP
#endif

#ifdef do_open
#undef do_open
#endif
#ifdef do_close
#undef do_close
#endif

#define NEED_sv_2pvbyte
#define NEED_newCONSTSUB
#define NEED_newRV_noinc
#define NEED_newSVpvn_flags
#define NEED_sv_2pv_flags
#include "ppport.h"

#include <sstream>
#include "ux.hpp"

{'fraction_non_alphanumeric': 0.097856477166822, 'fraction_numerical': 0.003727865796831314, 'mean_word_length': 6.458333333333333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5472711', 'n_tokens_mistral': 439, 'n_tokens_neox': 423, 'n_words': 114}