<commit_before>#ifndef UTILS_H_
#define UTILS_H_

// defines

#define KB 1024
#define MB (1024 * KB)

// number of elements in statically declared array
#define DIM(S_ARR_) (sizeof(S_ARR_) / sizeof(S_ARR_[0]))

#define MIN(A_, B_) ((A_) < (B_) ? (A_) : (B_))
#define MAX(A_, B_) ((A_) > (B_) ? (A_) : (B_))

// align value to N-byte boundary
#define ALIGN(VAL_, ALIGNMENT_) (((VAL_) + ((ALIGNMENT_) - 1)) & ~((ALIGNMENT_) - 1))

// read/write u32/16 big/little endian
#define read_u32_be(buf) (unsigned int)(((buf)[0] << 24) + ((buf)[1] << 16) + ((buf)[2] << 8) + ((buf)[3]))
#define read_u32_le(buf) (unsigned int)(((buf)[1] << 24) + ((buf)[0] << 16) + ((buf)[3] << 8) + ((buf)[2]))
#define write_u32_be(buf, val) do { \
   (buf)[0] = ((val) >> 24) & 0xFF; \
   (buf)[1] = ((val) >> 16) & 0xFF; \
   (buf)[2] = ((val) >> 8) & 0xFF; \
   (buf)[3] = (val) & 0xFF; \
} while(0)
#define read_u16_be(buf) (((buf)[0] << 8) + ((buf)[1]))
#define write_u16_be(buf, val) do { \
   (buf)[0] = ((val) >> 8) & 0xFF; \
   (buf)[1] = ((val)) & 0xFF; \
} while(0)

// print nibbles and bytes
#define print_nibble(NIB_) putchar((NIB_) < 10 ? ('0' + (NIB_)) : ('A' + (NIB_) - 0xA))
#define print_byte(BYTE_) do { \
    print_nibble((BYTE_) >> 4); \
    print_nibble((BYTE_) & 0x0F); \
    putchar(' '); \
  } while(0)

// global verbosity setting
extern int g_verbosity;

#define ERROR(...) fprintf(stderr, __VA_ARGS__)
#define INFO(...) if (g_verbosity) printf(__VA_ARGS__)
#define INFO_HEX(...) if (g_verbosity) print_hex(__VA_ARGS__)

// functions

// determine if value is power of 2
// returns 1 if val is power of 2, 0 otherwise
int is_power2(unsigned int val);

// print buffer as hex bytes
// buf: buffer to read bytes from
// length: length of buffer to print
void print_hex(unsigned char *buf, int length);

// perform byteswapping to convert from v64 to z64 ordering
void swap_bytes(unsigned char *data, long length);

// read entire contents of file into buffer
// returns file size or negative on error
long read_file(const char *file_name, unsigned char **data);

// write buffer to file
// returns number of bytes written out or -1 on failure
long write_file(const char *file_name, unsigned char *data, long length);
<commit_msg>Add mkdir() compatibility macro for Visual Studio<commit_after>#ifndef UTILS_H_
#define UTILS_H_

// defines

#define KB 1024
#define MB (1024 * KB)

// number of elements in statically declared array
#define DIM(S_ARR_) (sizeof(S_ARR_) / sizeof(S_ARR_[0]))

#define MIN(A_, B_) ((A_) < (B_) ? (A_) : (B_))
#define MAX(A_, B_) ((A_) > (B_) ? (A_) : (B_))

// align value to N-byte boundary
#define ALIGN(VAL_, ALIGNMENT_) (((VAL_) + ((ALIGNMENT_) - 1)) & ~((ALIGNMENT_) - 1))

// read/write u32/16 big/little endian
#define read_u32_be(buf) (unsigned int)(((buf)[0] << 24) + ((buf)[1] << 16) + ((buf)[2] << 8) + ((buf)[3]))
#define read_u32_le(buf) (unsigned int)(((buf)[1] << 24) + ((buf)[0] << 16) + ((buf)[3] << 8) + ((buf)[2]))
#define write_u32_be(buf, val) do { \
   (buf)[0] = ((val) >> 24) & 0xFF; \
   (buf)[1] = ((val) >> 16) & 0xFF; \
   (buf)[2] = ((val) >> 8) & 0xFF; \
   (buf)[3] = (val) & 0xFF; \
} while(0)
#define read_u16_be(buf) (((buf)[0] << 8) + ((buf)[1]))
#define write_u16_be(buf, val) do { \
   (buf)[0] = ((val) >> 8) & 0xFF; \
   (buf)[1] = ((val)) & 0xFF; \
} while(0)

// print nibbles and bytes
#define print_nibble(NIB_) putchar((NIB_) < 10 ? ('0' + (NIB_)) : ('A' + (NIB_) - 0xA))
#define print_byte(BYTE_) do { \
    print_nibble((BYTE_) >> 4); \
    print_nibble((BYTE_) & 0x0F); \
    putchar(' '); \
  } while(0)

// Visual Studio compatibility
#if defined _MSC_VER
  #include <direct.h>
  #define mkdir(DIR_, PERM_) _mkdir(DIR_)
#endif

// global verbosity setting
extern int g_verbosity;

#define ERROR(...) fprintf(stderr, __VA_ARGS__)
#define INFO(...) if (g_verbosity) printf(__VA_ARGS__)
#define INFO_HEX(...) if (g_verbosity) print_hex(__VA_ARGS__)

// functions

// determine if value is power of 2
// returns 1 if val is power of 2, 0 otherwise
int is_power2(unsigned int val);

// print buffer as hex bytes
// buf: buffer to read bytes from
// length: length of buffer to print
void print_hex(unsigned char *buf, int length);

// perform byteswapping to convert from v64 to z64 ordering
void swap_bytes(unsigned char *data, long length);

// read entire contents of file into buffer
// returns file size or negative on error
long read_file(const char *file_name, unsigned char **data);

// write buffer to file
// returns number of bytes written out or -1 on failure
long write_file(const char *file_name, unsigned char *data, long length);

{'fraction_non_alphanumeric': 0.18890814558058924, 'fraction_numerical': 0.036395147313691506, 'mean_word_length': 3.927427961579509, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '149406', 'n_tokens_mistral': 1844, 'n_tokens_neox': 1725, 'n_words': 533}