<commit_before>/* $Id$ */
/*
 * %PSC_START_COPYRIGHT%
 * -----------------------------------------------------------------------------
 * Copyright (c) 2007-2013, Pittsburgh Supercomputing Center (PSC).
 *
 * Permission to use, copy, and modify this software and its documentation
 * without fee for personal use or non-commercial use within your organization
 * is hereby granted, provided that the above copyright notice is preserved in
 * all copies and that the copyright and this permission notice appear in
 * supporting documentation.  Permission to redistribute this software to other
 * organizations or individuals is not permitted without the written permission
 * of the Pittsburgh Supercomputing Center.  PSC makes no representations about
 * the suitability of this software for any purpose.  It is provided "as is"
 * without express or implied warranty.
 * -----------------------------------------------------------------------------
 * %PSC_END_COPYRIGHT%
 */

/*
 * odtable: on-disk table for persistent storage of otherwise memory
 * resident data structures.
 */

#ifndef _PFL_ODTABLE_H_
#define _PFL_ODTABLE_H_

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>

#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>

#include "pfl/list.h"
#include "pfl/lockedlist.h"
#include "pfl/vbitmap.h"
#include "psc_util/crc.h"
#include "psc_util/lock.h"
#include "psc_util/log.h"

#define ODT_DEFAULT_ITEM_SIZE	128
#define	ODT_DEFAULT_TABLE_SIZE	(1024 * 128)

#define ODTBL_VERS		UINT64_C(0x0000000000000002)
#define ODTBL_MAGIC		UINT64_C(0x001122335577bbdd)
#define ODTBL_INUSE		UINT64_C(0xf0f0f0f0f0f0f0f0)
#define ODTBL_FREE		UINT64_C(0xffffffffffffffff)
#define ODTBL_BAD		UINT64_C(0x1010101010101010)
#define ODTBL_ERR		((size_t)-1)

enum od_table_errors {
	ODTBL_NO_ERR	= 0,
	ODTBL_MAGIC_ERR	= 1,
	ODTBL_SLOT_ERR	= 2,
	ODTBL_INUSE_ERR	= 3,
	ODTBL_FREE_ERR	= 4,
	ODTBL_CRC_ERR	= 5,
	ODTBL_BADSL_ERR	= 6,
	ODTBL_KEY_ERR	= 7
};

/* Table entries start on a 4k boundary */
#define ODTBL_START		0x1000

struct odtable_hdr {
	size_t			 odth_nelems;
	size_t			 odth_elemsz;	/* does not include odtable_entftr */
	size_t			 odth_slotsz;	/* does include odtable_entftr */
	uint64_t		 odth_magic;
	uint32_t		 odth_version;
	uint32_t		 odth_options;	/* see ODTBL_OPT_* below */
	off_t			 odth_start;
} __packed;

/* odtable options */
#define ODTBL_OPT_CRC		(1 << 0)
#define ODTBL_OPT_SYNC		(1 << 1)

/* entry footer */
struct odtable_entftr {
	uint64_t		 odtf_crc;
	uint64_t		 odtf_inuse;
	uint64_t		 odtf_slotno;
	uint64_t		 odtf_magic;
#define odtf_key odtf_crc
};

#define ODT_NAME_MAX		16
#define ODT_DEFAULT_ITEM_SIZE	128

struct odtable {
	struct psc_vbitmap	*odt_bitmap;
	void			*odt_base;
	struct odtable_hdr	*odt_hdr;
	psc_spinlock_t		 odt_lock;
	int			 odt_fd;
	void			*odt_handle;		/* need this for odtable in ZFS */
	char			 odt_name[ODT_NAME_MAX];
	struct psclist_head	 odt_lentry;
};

struct odtable_receipt {
	size_t			 odtr_elem;
	uint64_t		 odtr_key;
};

#define ODTABLE_MAPSZ(odt)	_odtable_getitemoff((odt), (odt)->odt_hdr->odth_nelems, 1)

#define ODTBL_FLG_RDONLY	(1 << 0)

struct odtable_receipt *
	 odtable_putitem(struct odtable *, void *, size_t);
int	 odtable_create(const char *, size_t, size_t, int, int);
int	 odtable_freeitem(struct odtable *, struct odtable_receipt *);<commit_msg>include more information about odtable in debug messages<commit_after>/* $Id$ */
/*
 * %PSC_START_COPYRIGHT%
 * -----------------------------------------------------------------------------
 * Copyright (c) 2007-2013, Pittsburgh Supercomputing Center (PSC).
 *
 * Permission to use, copy, and modify this software and its documentation
 * without fee for personal use or non-commercial use within your organization
 * is hereby granted, provided that the above copyright notice is preserved in
 * all copies and that the copyright and this permission notice appear in
 * supporting documentation.  Permission to redistribute this software to other
 * organizations or individuals is not permitted without the written permission
 * of the Pittsburgh Supercomputing Center.  PSC makes no representations about
 * the suitability of this software for any purpose.  It is provided "as is"
 * without express or implied warranty.
 * -----------------------------------------------------------------------------
 * %PSC_END_COPYRIGHT%
 */

/*
 * odtable: on-disk table for persistent storage of otherwise memory
 * resident data structures.
 */

#ifndef _PFL_ODTABLE_H_
#define _PFL_ODTABLE_H_

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>

#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>

#include "pfl/list.h"
#include "pfl/lockedlist.h"
#include "pfl/vbitmap.h"
#include "psc_util/crc.h"
#include "psc_util/lock.h"
#include "psc_util/log.h"

#define ODT_DEFAULT_ITEM_SIZE	128
#define	ODT_DEFAULT_TABLE_SIZE	(1024 * 128)

#define ODTBL_VERS		UINT64_C(0x0000000000000002)
#define ODTBL_MAGIC		UINT64_C(0x001122335577bbdd)
#define ODTBL_INUSE		UINT64_C(0xf0f0f0f0f0f0f0f0)
#define ODTBL_FREE		UINT64_C(0xffffffffffffffff)
#define ODTBL_BAD		UINT64_C(0x1010101010101010)
#define ODTBL_ERR		((size_t)-1)

enum od_table_errors {
	ODTBL_NO_ERR	= 0,
	ODTBL_MAGIC_ERR	= 1,
	ODTBL_SLOT_ERR	= 2,
	ODTBL_INUSE_ERR	= 3,
	ODTBL_FREE_ERR	= 4,
	ODTBL_CRC_ERR	= 5,
	ODTBL_BADSL_ERR	= 6,
	ODTBL_KEY_ERR	= 7
};

/* Table entries start on a 4k boundary */
#define ODTBL_START		0x1000

struct odtable_hdr {
	size_t			 odth_nelems;
	size_t			 odth_elemsz;	/* does not include odtable_entftr */
	size_t			 odth_slotsz;	/* does include odtable_entftr */
	uint64_t		 odth_magic;
	uint32_t		 odth_version;
	uint32_t		 odth_options;	/* see ODTBL_OPT_* below */
	off_t			 odth_start;
} __packed;

/* odtable options */
#define ODTBL_OPT_CRC		(1 << 0)
#define ODTBL_OPT_SYNC		(1 << 1)

/* entry footer */
struct odtable_entftr {
	uint64_t		 odtf_crc;
	uint64_t		 odtf_inuse;
	uint64_t		 odtf_slotno;
	uint64_t		 odtf_magic;
#define odtf_key odtf_crc
};

#define ODT_NAME_MAX		16
#define ODT_DEFAULT_ITEM_SIZE	128

struct odtable {
	struct psc_vbitmap	*odt_bitmap;
	void			*odt_base;
	struct odtable_hdr	*odt_hdr;
	psc_spinlock_t		 odt_lock;
	int			 odt_fd;
	void			*odt_handle;		/* need this for odtable in ZFS */
	char			 odt_name[ODT_NAME_MAX];
	struct psclist_head	 odt_lentry;
};

#define DPRINTF_ODT(lvl, odt, fmt, ...)					\
	psclog((lvl), "odt@%p[%s] " fmt,				\
	    (odt), (odt)->odt_name, ## __VA_ARGS__)

struct odtable_receipt {
	size_t			 odtr_elem;
	uint64_t		 odtr_key;
};

#define ODTABLE_MAPSZ(odt)	_odtable_getitemoff((odt), (odt)->odt_hdr->odth_nelems, 1)

#define ODTBL_FLG_RDONLY	(1 << 0)

struct odtable_receipt *
	 odtable_putitem(struct odtable *, void *, size_t);
int	 odtable_create(const char *, size_t, size_t, int, int);
int	 odtable_freeitem(struct odtable *, struct odtable_receipt *);
{'fraction_non_alphanumeric': 0.13290585835033517, 'fraction_numerical': 0.03730690760711163, 'mean_word_length': 3.6591989137813985, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6863070', 'n_tokens_mistral': 2873, 'n_tokens_neox': 2623, 'n_words': 647}