<commit_before>#ifndef __omx_lib_h__
#define __omx_lib_h__

#include <string.h>
#include <stdio.h>
#include <stdint.h>

#include "open-mx.h"
#include "omx_io.h"
#include "omx_list.h"
#include "omx__valgrind.h"

/************
 * Debugging
 */

#ifdef OMX_DEBUG
#define omx__debug_assert(x) assert(x)
#define omx__debug_instr(x) do { x; } while (0)
#define omx__debug_printf(args...) fprintf(stderr, args)
#else
#define omx__debug_assert(x) /* nothing */
#define omx__debug_instr(x) /* nothing */
#define omx__debug_printf(args...) /* nothing */
#endif

/*****************
 * Various macros
 */

#define OMX_DEVNAME "/dev/openmx"
/* FIXME: envvar to configure? */
<commit_msg>update the lib to use /dev/open-mx<commit_after>#ifndef __omx_lib_h__
#define __omx_lib_h__

#include <string.h>
#include <stdio.h>
#include <stdint.h>

#include "open-mx.h"
#include "omx_io.h"
#include "omx_list.h"
#include "omx__valgrind.h"

/************
 * Debugging
 */

#ifdef OMX_DEBUG
#define omx__debug_assert(x) assert(x)
#define omx__debug_instr(x) do { x; } while (0)
#define omx__debug_printf(args...) fprintf(stderr, args)
#else
#define omx__debug_assert(x) /* nothing */
#define omx__debug_instr(x) /* nothing */
#define omx__debug_printf(args...) /* nothing */
#endif

/*****************
 * Various macros
 */

#define OMX_DEVNAME "/dev/open-mx"
/* FIXME: envvar to configure? */

{'fraction_non_alphanumeric': 0.19416058394160585, 'fraction_numerical': 0.00145985401459854, 'mean_word_length': 5.066371681415929, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5733323', 'n_tokens_mistral': 542, 'n_tokens_neox': 539, 'n_words': 113}