<commit_before>/* $Id$ */

#ifndef _PFL_JOURNAL_H_
#define _PFL_JOURNAL_H_

#include <sys/user.h>

#include "psc_ds/dynarray.h"
#include "psc_util/atomic.h"
#include "psc_util/lock.h"
#include "psc_util/thread.h"
#include "psc_util/waitq.h"

#define PJ_LOCK(pj)     spinlock(&(pj)->pj_lock)
#define PJ_ULOCK(pj)    freelock(&(pj)->pj_lock)

#define PJH_MAGIC	0x45678912aabbccffULL	/* magic number of the journal header */

#define PJH_VERSION     0x01

struct _psc_journal_hdr {			
	int32_t		_pjh_entsz;
	int32_t		_pjh_nents;
	uint32_t	_pjh_version;
	uint32_t	_pjh_options;
	int32_t		_pjh_readahead;
	uint32_t	_pjh_unused;
	uint64_t	_pjh_start_off;
	uint64_t	_pjh_magic;		
	uint64_t	_pjh_chksum;		/* keep it last and aligned at a 8 byte boundary */
};

#define PJH_ALIGN_SIZE	PSC_ALIGN(sizeof(struct _psc_journal_hdr), PAGE_SIZE)

struct psc_journal_hdr {
	struct _psc_journal_hdr pjh;
	char			pjg__pad[PJH_ALIGN_SIZE - sizeof(struct _psc_journal_hdr)];
#define pjh_entsz	pjh._pjh_entsz
#define pjh_nents	pjh._pjh_nents
#define pjh_version	pjh._pjh_version
#define pjh_options	pjh._pjh_options
#define pjh_readahead	pjh._pjh_readahead
#define pjh_unused	pjh._pjh_unused
#define pjh_start_off	pjh._pjh_start_off
#define pjh_magic	pjh._pjh_magic
#define pjh_chksum	pjh._pjh_chksum
};

#define	MAX_NUM_PJBUF		 8		/* number of journal buffers to keep around */

#define PJ_NONE			(0 << 0)
#define PJ_WANTBUF		(1 << 0)
#define PJ_WANTSLOT		(1 << 1)

struct psc_journal {
	psc_spinlock_t		 pj_lock;	/* contention lock */
	int			 pj_fd;		/* open file descriptor to disk */
	char			*pj_logname;	/* log file name */
	uint64_t		 pj_nextxid;	/* next transaction ID */
	int32_t		 	 pj_nextwrite;	/* next entry slot to write to */
	struct psclist_head	 pj_pndgxids;
	struct dynarray		 pj_bufs;
	struct psc_journal_hdr	*pj_hdr;
	struct psc_waitq	 pj_waitq;
	int			 pj_flags;
};

typedef void (*psc_jhandler)(struct dynarray *, int *);

/*
 * Start offset to write journal entries (i.e., pje). This means that the log header must be smaller than PJE_OFFSET.
 */
#define PJE_OFFSET		PJH_ALIGN_SIZE

#define PJE_XID_NONE		0			/* invalid transaction ID */
#define PJE_MAGIC		0x45678912aabbccddULL	/* magic number for each journal entry */

/*
 * Journal entry types - higher bits after PJET_LASTBIT are used to identify different log users.
 */
#define PJE_NONE		(0 << 0)		/* null journal record */
#define PJE_FORMAT		(1 << 0)		/* newly-formatted journal record */
#define PJE_XCLOSED		(1 << 1)		/* xid is closed */
#define PJE_XSTARTED		(1 << 2)		/* transaction began */
#define PJE_STARTUP		(1 << 3)		/* system startup */
#define PJE_LASTBIT		 3			/* denote the last used bit */

/*
 * psc_journal_enthdr - journal entry header.
 * @pje_magic: validity check.
 * @pje_type: app-specific log entry type.
 * @pje_xid: journal transaction id.
 * @pje_sid: xid sub-id.
 * @pje_chksum: simple XOR checksum
 * @pje_data: application data.
 * Notes: at some point we may want to make this into a footer which has
 *    a crc field.
 */
struct psc_journal_enthdr {
	uint64_t		pje_magic;
	uint64_t		pje_xid;
	uint32_t		pje_type;		/* see above */
	uint32_t		pje_sid;
	/* 
	 * This field is used to calculate the CRC checksum of the following data.
	 * It also indicates if the log entry is a special one (i.e., one without
	 * payload).
	 */
	uint64_t		pje_chksum;
	uint32_t		pje_len;		
	/*
	 * The length of the pje_data[0] is also embedded and can be figured out
	 * by log replay functions.
	 */
	char			pje_data[0];
} __packed;

#define	PJ_PJESZ(p)		((p)->pj_hdr->pjh_entsz)<commit_msg>adjust position of checksum<commit_after>/* $Id$ */

#ifndef _PFL_JOURNAL_H_
#define _PFL_JOURNAL_H_

#include <sys/user.h>

#include "psc_ds/dynarray.h"
#include "psc_util/atomic.h"
#include "psc_util/lock.h"
#include "psc_util/thread.h"
#include "psc_util/waitq.h"

#define PJ_LOCK(pj)     spinlock(&(pj)->pj_lock)
#define PJ_ULOCK(pj)    freelock(&(pj)->pj_lock)

#define PJH_MAGIC	0x45678912aabbccffULL	/* magic number of the journal header */

#define PJH_VERSION     0x01

struct _psc_journal_hdr {			
	int32_t		_pjh_entsz;
	int32_t		_pjh_nents;
	uint32_t	_pjh_version;
	uint32_t	_pjh_options;
	int32_t		_pjh_readahead;
	uint32_t	_pjh_unused;
	uint64_t	_pjh_start_off;
	uint64_t	_pjh_magic;		
	uint64_t	_pjh_chksum;		/* keep it last and aligned at a 8 byte boundary */
};

#define PJH_ALIGN_SIZE	PSC_ALIGN(sizeof(struct _psc_journal_hdr), PAGE_SIZE)

struct psc_journal_hdr {
	struct _psc_journal_hdr pjh;
	char			pjg__pad[PJH_ALIGN_SIZE - sizeof(struct _psc_journal_hdr)];
#define pjh_entsz	pjh._pjh_entsz
#define pjh_nents	pjh._pjh_nents
#define pjh_version	pjh._pjh_version
#define pjh_options	pjh._pjh_options
#define pjh_readahead	pjh._pjh_readahead
#define pjh_unused	pjh._pjh_unused
#define pjh_start_off	pjh._pjh_start_off
#define pjh_magic	pjh._pjh_magic
#define pjh_chksum	pjh._pjh_chksum
};

#define	MAX_NUM_PJBUF		 8		/* number of journal buffers to keep around */

#define PJ_NONE			(0 << 0)
#define PJ_WANTBUF		(1 << 0)
#define PJ_WANTSLOT		(1 << 1)

struct psc_journal {
	psc_spinlock_t		 pj_lock;	/* contention lock */
	int			 pj_fd;		/* open file descriptor to disk */
	char			*pj_logname;	/* log file name */
	uint64_t		 pj_nextxid;	/* next transaction ID */
	int32_t		 	 pj_nextwrite;	/* next entry slot to write to */
	struct psclist_head	 pj_pndgxids;
	struct dynarray		 pj_bufs;
	struct psc_journal_hdr	*pj_hdr;
	struct psc_waitq	 pj_waitq;
	int			 pj_flags;
};

typedef void (*psc_jhandler)(struct dynarray *, int *);

/*
 * Start offset to write journal entries (i.e., pje). This means that the log header must be smaller than PJE_OFFSET.
 */
#define PJE_OFFSET		PJH_ALIGN_SIZE

#define PJE_XID_NONE		0			/* invalid transaction ID */
#define PJE_MAGIC		0x45678912aabbccddULL	/* magic number for each journal entry */

/*
 * Journal entry types - higher bits after PJET_LASTBIT are used to identify different log users.
 */
#define PJE_NONE		(0 << 0)		/* null journal record */
#define PJE_FORMAT		(1 << 0)		/* newly-formatted journal record */
#define PJE_XCLOSED		(1 << 1)		/* xid is closed */
#define PJE_XSTARTED		(1 << 2)		/* transaction began */
#define PJE_STARTUP		(1 << 3)		/* system startup */
#define PJE_LASTBIT		 3			/* denote the last used bit */

/*
 * psc_journal_enthdr - journal entry header.
 * @pje_magic: validity check.
 * @pje_type: app-specific log entry type.
 * @pje_xid: journal transaction id.
 * @pje_sid: xid sub-id.
 * @pje_chksum: simple XOR checksum
 * @pje_data: application data.
 * Notes: at some point we may want to make this into a footer which has
 *    a crc field.
 */
struct psc_journal_enthdr {
	uint64_t		pje_magic;
	uint32_t		pje_type;		/* see above */
	uint32_t		pje_padding;
	uint64_t		pje_xid;
	uint32_t		pje_sid;
	/* 
	 * This field is used to calculate the CRC checksum of the following data.
	 * It also indicates if the log entry is a special one (i.e., one without
	 * payload).
	 */
	uint32_t		pje_len;		
	uint64_t		pje_chksum;		/* last field before data */
	/*
	 * The length of the pje_data[0] is also embedded and can be figured out
	 * by log replay functions.
	 */
	char			pje_data[0];
} __packed;

#define	PJ_PJESZ(p)		((p)->pj_hdr->pjh_entsz)
{'fraction_non_alphanumeric': 0.09738816337871631, 'fraction_numerical': 0.021672686857460405, 'mean_word_length': 3.954576737783895, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5384328', 'n_tokens_mistral': 3251, 'n_tokens_neox': 3010, 'n_words': 772}