<commit_before>#define yy_create_buffer bx_create_buffer
#define yy_delete_buffer bx_delete_buffer
#define yy_scan_buffer bx_scan_buffer
#define yy_scan_string bx_scan_string
#define yy_scan_bytes bx_scan_bytes
#define yy_flex_debug bx_flex_debug
#define yy_init_buffer bx_init_buffer
#define yy_flush_buffer bx_flush_buffer
#define yy_load_buffer_state bx_load_buffer_state
#define yy_switch_to_buffer bx_switch_to_buffer
#define yyin bxin
#define yyleng bxleng
#define yylex bxlex
#define yyout bxout
#define yyrestart bxrestart
#define yytext bxtext
#define yywrap bxwrap

/* A lexical scanner generated by flex */

/* Scanner skeleton version:
 * $Header$
 */

#define FLEX_SCANNER
#define YY_FLEX_MAJOR_VERSION 2
#define YY_FLEX_MINOR_VERSION 5

#include <stdio.h>


/* cfront 1.2 defines "c_plusplus" instead of "__cplusplus" */
#ifdef c_plusplus
#ifndef __cplusplus
#define __cplusplus
#endif
#endif

<commit_msg>Made to compile on MacOS9<commit_after>#define yy_create_buffer bx_create_buffer
#define yy_delete_buffer bx_delete_buffer
#define yy_scan_buffer bx_scan_buffer
#define yy_scan_string bx_scan_string
#define yy_scan_bytes bx_scan_bytes
#define yy_flex_debug bx_flex_debug
#define yy_init_buffer bx_init_buffer
#define yy_flush_buffer bx_flush_buffer
#define yy_load_buffer_state bx_load_buffer_state
#define yy_switch_to_buffer bx_switch_to_buffer
#define yyin bxin
#define yyleng bxleng
#define yylex bxlex
#define yyout bxout
#define yyrestart bxrestart
#define yytext bxtext
#define yywrap bxwrap

/* A lexical scanner generated by flex */

/* Scanner skeleton version:
 * $Header$
 */

#define FLEX_SCANNER
#define YY_FLEX_MAJOR_VERSION 2
#define YY_FLEX_MINOR_VERSION 5

#include <stdio.h>
#include <unistd.h>


/* cfront 1.2 defines "c_plusplus" instead of "__cplusplus" */
#ifdef c_plusplus
#ifndef __cplusplus
#define __cplusplus
#endif
#endif


{'fraction_non_alphanumeric': 0.05876068376068376, 'fraction_numerical': 0.004807692307692308, 'mean_word_length': 7.214912280701754, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6225664', 'n_tokens_mistral': 720, 'n_tokens_neox': 729, 'n_words': 178}