<commit_before>#ifndef _J64_H_
#define _J64_H_

/*
 * J64 library type definitions, macros and function declarations.
 * This library stores every JSON value as a 8-byte word, where
 * the least significant bits indicate the type of the value and
 * whether the word contains the direct value (immediate) or contains
 * a pointer to a heap-allocated value (boxed).
 * 
 * Immediate values can include:
 *   - literal values, which include:
 *     - null
 *     - true, false
 *     - empty string, object or array
 *   - 62-bit integers
 *   - 61-bit floating point numbers
 *     (with 3 least significant bits in mantissa zeroed)
 *   - short strings (with length less or equal to 7 bytes)
 *
 * Boxed values can include:
 *   - long strings (with length over 7 bytes)
 *   - non-empty arrays
 *   - non-empty objects
 */

#include <stdint.h>
#include <stddef.h>

typedef uint64_t	 _j64_word_t;
typedef int64_t		 _j64_int_t;
typedef double		 _j64_float_t;
typedef uint8_t		 _j64_byte_t;
typedef void		*_j64_ptr_t;

/*
 * J64 union type. The unsigned integer word (w) is
 * used to perform bit manipulations, while other fields
 * are used for differnet types of accesses.
 */
typedef union {
	_j64_word_t	w;
	_j64_int_t	i;
	_j64_float_t	f;
	_j64_byte_t	b[8];
	_j64_ptr_t	p;
} j64_t;

#define _J64_INITIALIZER {0}

/* 
 * The three least significant bits form a primary tag that denotes
 * how the next 61 bits should be interpreted, which can be one
 * of the following:
 *   - immediate literal
 *   - immediate integer
 *   - immediate floating-point number
 *   - immediate short string
 *   - boxed long string
 *   - boxed array
 *   - boxed object
 *
 * Two bit sequences denote an integer, which are 011 (3) and 111 (7).
 * This way we can get one free extra bit when storing integers (thanks vihtori).
 *
 * Storing the primary tag with 3 bits while storing full pointers in the word
 * at the same time rely on the system memory allocator returning pointers
 * that are aligned on at least an 8-byte boundary. Otherwise this library
 * won't function.
 */
#define _J64_TAG_PRIM_SIZE	3
#define _J64_TAG_PRIM_MASK	((1 << _J64_TAG_PRIM_SIZE) - 1)
#define _J64_TAG_PRIM_INT_SIZE	(_J64_TAG_PRIM_SIZE - 1)
#define _J64_TAG_PRIM_INT_MASK	((1 << _J64_TAG_PRIM_INT_SIZE) - 1)

#define J64_TAG_PRIM_LIT	0x0
#define J64_TAG_PRIM_FLOAT	0x1
#define J64_TAG_PRIM_ISTR	0x2
#define J64_TAG_PRIM_INT0	0x3
#define J64_TAG_PRIM_BSTR	0x4
#define J64_TAG_PRIM_ARR	0x5
#define J64_TAG_PRIM_OBJ	0x6
#define J64_TAG_PRIM_INT1	0x7

#define j64_prim_tag(j)		((j).w & _J64_TAG_PRIM_MASK)
#define _j64_prim_tag_set(j, t)	((j).w |= (t))

#define _j64_get_ptr(j)		((_j64_ptr_t)((j).w & ~_J64_TAG_PRIM_MASK))

#define J64_INT_MIN		(-0x1fffffffffffffffLL - 1)
#define J64_INT_MAX		0x1fffffffffffffffLL

#define j64_is_immed(j)							\
	(j64_prim_tag(j) <= J64_TAG_PRIM_INT0 ||			\
	 j64_prim_tag(j) == J64_TAG_PRIM_INT1)
#define j64_is_boxed(j)							\
	(j64_prim_tag(j) >= J64_TAG_PRIM_BSTR &&			\
	 j64_prim_tag(j) <= J64_TAG_PRIM_OBJ)

/*
 * 3 bits after a primary tag of J64_TAG_PRIM_LIT form a literal tag,
 * which denotes the type of a literal value stored. A literal value
 * can be one of the following:
 *   - null<commit_msg>Remove j64_is_immed/boxed -macros for now<commit_after>#ifndef _J64_H_
#define _J64_H_

/*
 * J64 library type definitions, macros and function declarations.
 * This library stores every JSON value as a 8-byte word, where
 * the least significant bits indicate the type of the value and
 * whether the word contains the direct value (immediate) or contains
 * a pointer to a heap-allocated value (boxed).
 * 
 * Immediate values can include:
 *   - literal values, which include:
 *     - null
 *     - true, false
 *     - empty string, object or array
 *   - 62-bit integers
 *   - 61-bit floating point numbers
 *     (with 3 least significant bits in mantissa zeroed)
 *   - short strings (with length less or equal to 7 bytes)
 *
 * Boxed values can include:
 *   - long strings (with length over 7 bytes)
 *   - non-empty arrays
 *   - non-empty objects
 */

#include <stdint.h>
#include <stddef.h>

typedef uint64_t	 _j64_word_t;
typedef int64_t		 _j64_int_t;
typedef double		 _j64_float_t;
typedef uint8_t		 _j64_byte_t;
typedef void		*_j64_ptr_t;

/*
 * J64 union type. The unsigned integer word (w) is
 * used to perform bit manipulations, while other fields
 * are used for differnet types of accesses.
 */
typedef union {
	_j64_word_t	w;
	_j64_int_t	i;
	_j64_float_t	f;
	_j64_byte_t	b[8];
	_j64_ptr_t	p;
} j64_t;

#define _J64_INITIALIZER {0}

/* 
 * The three least significant bits form a primary tag that denotes
 * how the next 61 bits should be interpreted, which can be one
 * of the following:
 *   - immediate literal
 *   - immediate integer
 *   - immediate floating-point number
 *   - immediate short string
 *   - boxed long string
 *   - boxed array
 *   - boxed object
 *
 * Two bit sequences denote an integer, which are 011 (3) and 111 (7).
 * This way we can get one free extra bit when storing integers (thanks vihtori).
 *
 * Storing the primary tag with 3 bits while storing full pointers in the word
 * at the same time rely on the system memory allocator returning pointers
 * that are aligned on at least an 8-byte boundary. Otherwise this library
 * won't function.
 */
#define _J64_TAG_PRIM_SIZE	3
#define _J64_TAG_PRIM_MASK	((1 << _J64_TAG_PRIM_SIZE) - 1)
#define _J64_TAG_PRIM_INT_SIZE	(_J64_TAG_PRIM_SIZE - 1)
#define _J64_TAG_PRIM_INT_MASK	((1 << _J64_TAG_PRIM_INT_SIZE) - 1)

#define J64_TAG_PRIM_LIT	0x0
#define J64_TAG_PRIM_FLOAT	0x1
#define J64_TAG_PRIM_ISTR	0x2
#define J64_TAG_PRIM_INT0	0x3
#define J64_TAG_PRIM_BSTR	0x4
#define J64_TAG_PRIM_ARR	0x5
#define J64_TAG_PRIM_OBJ	0x6
#define J64_TAG_PRIM_INT1	0x7

#define j64_prim_tag(j)		((j).w & _J64_TAG_PRIM_MASK)
#define _j64_prim_tag_set(j, t)	((j).w |= (t))

#define _j64_get_ptr(j)		((_j64_ptr_t)((j).w & ~_J64_TAG_PRIM_MASK))

#define J64_INT_MIN		(-0x1fffffffffffffffLL - 1)
#define J64_INT_MAX		0x1fffffffffffffffLL

/*
 * 3 bits after a primary tag of J64_TAG_PRIM_LIT form a literal tag,
 * which denotes the type of a literal value stored. A literal value
 * can be one of the following:
 *   - null
{'fraction_non_alphanumeric': 0.07971713275474124, 'fraction_numerical': 0.04789456766313083, 'mean_word_length': 3.7795698924731185, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '540878', 'n_tokens_mistral': 2528, 'n_tokens_neox': 2311, 'n_words': 749}