<commit_before>#ifndef __PORTABLE_H__
#define __PORTABLE_H__

extern int debugmode;

#ifndef WIN32
#define dbg_printf(fmt, args...)	do{ if(debugmode) fprintf(stderr, fmt, ## args); } while(0)
#define dbg_dump_long(src, cnt, addr, skip) do{ if(debugmode) dump_long(src, cnt, addr, skip); } while(0)
#else

#ifdef DEBUG
#define dbg_printf(fmt, ...)	fprintf(stderr, fmt, __VA_ARGS__)
#define dbg_dump_long(src, cnt, addr, skip) dump_long(src, cnt, addr, skip)
#else
#define dbg_dump_long(src, cnt, addr, skip)
#define dbg_printf(fmt, ...)    /* Don't do anything in release builds */
#endif
#endif

#ifndef _MSC_VER
#include <unistd.h>
#endif
#ifdef WIN32
#include <Windows.h>
#include <direct.h>
#include <io.h>
#endif
#ifdef __linux__
#include <linux/limits.h>
#endif
#ifdef __APPLE__
#include <sys/syslimits.h>
#endif
#ifdef __FreeBSD__
#include <sys/param.h>
#endif

#ifndef WIN32
#define PATH_SEPARATOR '/'
#define msleep(ms) usleep((ms) * 1000)
#else
#define PATH_MAX MAX_PATH
#define PATH_SEPARATOR '\\'
#define msleep(ms) Sleep(ms)
#endif

#ifdef _MSC_VER
#define R_OK 04

#define open(filename,oflag)	_open(filename,oflag)
#define write(fd,buffer,count)	_write(fd,buffer,count)
#define read(fd,buffer,count)	_read(fd,buffer,count)
#define close(fd)				_close(fd)
#define access(filename,oflag)	_access(filename,oflag)
#define getcwd(buffer, maxlen)	_getcwd(buffer, maxlen)
#endif

#ifdef __GNUC__
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define BE32(x) __builtin_bswap32(x)
#define BE16(x) \
({ \
	typeof(x) __x = (x); \
	(((__x << 8) & 0xffff) | ((__x >> 8) & 0xff)); \
})
#else
#define BE32(x) x
#define BE16(x) x
#endif
#elif _MSC_VER // assume little endian...
#define BE32(x) _byteswap_ulong(x)
#define BE16(x) _byteswap_ushort(x)
#endif

#endif /* __PORTABLE_H__ */<commit_msg>portable: use __builtin_bswap16 for BE16 if >= 4.8 GNUC<commit_after>#ifndef __PORTABLE_H__
#define __PORTABLE_H__

extern int debugmode;

#ifndef WIN32
#define dbg_printf(fmt, args...)	do{ if(debugmode) fprintf(stderr, fmt, ## args); } while(0)
#define dbg_dump_long(src, cnt, addr, skip) do{ if(debugmode) dump_long(src, cnt, addr, skip); } while(0)
#else

#ifdef DEBUG
#define dbg_printf(fmt, ...)	fprintf(stderr, fmt, __VA_ARGS__)
#define dbg_dump_long(src, cnt, addr, skip) dump_long(src, cnt, addr, skip)
#else
#define dbg_dump_long(src, cnt, addr, skip)
#define dbg_printf(fmt, ...)    /* Don't do anything in release builds */
#endif
#endif

#ifndef _MSC_VER
#include <unistd.h>
#endif
#ifdef WIN32
#include <Windows.h>
#include <direct.h>
#include <io.h>
#endif
#ifdef __linux__
#include <linux/limits.h>
#endif
#ifdef __APPLE__
#include <sys/syslimits.h>
#endif
#ifdef __FreeBSD__
#include <sys/param.h>
#endif

#ifndef WIN32
#define PATH_SEPARATOR '/'
#define msleep(ms) usleep((ms) * 1000)
#else
#define PATH_MAX MAX_PATH
#define PATH_SEPARATOR '\\'
#define msleep(ms) Sleep(ms)
#endif

#ifdef _MSC_VER
#define R_OK 04

#define open(filename,oflag)	_open(filename,oflag)
#define write(fd,buffer,count)	_write(fd,buffer,count)
#define read(fd,buffer,count)	_read(fd,buffer,count)
#define close(fd)				_close(fd)
#define access(filename,oflag)	_access(filename,oflag)
#define getcwd(buffer, maxlen)	_getcwd(buffer, maxlen)
#endif

#ifdef __GNUC__
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define BE32(x) __builtin_bswap32(x)
#if (__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 8))
#define BE16(x) __builtin_bswap16(x)
#else
#define BE16(x) \
({ \
	typeof(x) __x = (x); \
	(((__x << 8) & 0xffff) | ((__x >> 8) & 0xff)); \
})
#endif
#else
#define BE32(x) x
#define BE16(x) x
#endif
#elif _MSC_VER // assume little endian...
#define BE32(x) _byteswap_ulong(x)
#define BE16(x) _byteswap_ushort(x)
#endif

#endif /* __PORTABLE_H__ */
{'fraction_non_alphanumeric': 0.15612489991993594, 'fraction_numerical': 0.020549773151854817, 'mean_word_length': 6.005607476635514, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '590290', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1542, 'n_words': 373}