<commit_before>#include <sys/types.h>
#include <sys/stat.h>

#include <fcntl.h>
#include <unistd.h>
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <time.h>
#include <stdio.h>

#include "base.h"
#include "log.h"
#include "buffer.h"
#include "response.h"
#include "file_cache_funcs.h"

#include "plugin.h"

#include "crc32.h"

#include "config.h"

#include "chunk_funcs.h"

#if defined HAVE_ZLIB_H && defined HAVE_LIBZ
# define USE_ZLIB
# include <zlib.h>
#endif

#if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2
# define USE_BZ2LIB
/* we don't need stdio interface */
# define BZ_NO_STDIO<commit_msg>added missing header<commit_after>#include <sys/types.h>
#include <sys/stat.h>

#include <fcntl.h>
#include <unistd.h>
#include <ctype.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <time.h>
#include <stdio.h>

#include "base.h"
#include "log.h"
#include "buffer.h"
#include "response.h"
#include "file_cache_funcs.h"

#include "plugin.h"

#include "crc32.h"

#include "config.h"

#include "chunk_funcs.h"

#if defined HAVE_ZLIB_H && defined HAVE_LIBZ
# define USE_ZLIB
# include <zlib.h>
#endif

#if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2
# define USE_BZ2LIB
/* we don't need stdio interface */
# define BZ_NO_STDIO
{'fraction_non_alphanumeric': 0.15668202764976957, 'fraction_numerical': 0.006144393241167435, 'mean_word_length': 5.264423076923077, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1830937', 'n_tokens_mistral': 558, 'n_tokens_neox': 546, 'n_words': 127}