<commit_before>/*
 * Copyright 2011-2012 the Redfish authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "common/config/mstorc.h"
#include "core/glitch_log.h"
#include "jorm/jorm_const.h"
#include "mds/defaults.h"
#include "mds/limits.h"
#include "mds/mstor.h"
#include "mds/user.h"
#include "msg/generic.h"
#include "util/error.h"
#include "util/fast_log.h"
#include "util/packed.h"
#include "util/path.h"
#include "util/simple_io.h"
#include "util/string.h"

#include <errno.h>
#include <inttypes.h>
#include <leveldb/c.h>
#include <pthread.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

/* leveldb storage scheme:
 * for file and directory nodes:
 *	n[8-byte node-id] => mnode
 * for files:
 *      f[8-byte node-id][8-byte offset] => 8-byte chunk ID
 * for directory children:
 *	c[8-byte node-id][child-name] => 8-byte child ID
 * for chunks:
 *      h[8-byte-chunk-id] => <packed-array of 4-byte OSD-IDs>
 * for zombie chunks:
 *      z[8-byte-death-time][8-byte-zombie-chunk-id] => []
 */
/****************************** constants ********************************/

#define MSTOR_CUR_VERSION 0x000000001U
#define MSTOR_VERSION_MAGIC "Fish"
#define MSTOR_VERSION_MAGIC_LEN 4
#define MSTOR_VERSION_BODY_LEN 8
#define MSTOR_VERSION_INVAL 0xffffffffU

/** TODO: allocate chunk IDs in a per-delegation fashion */
#define MSTOR_INIT_CID 1

#define MSTOR_ROOT_NID 0

#define MSTOR_PERM_EXEC 01
#define MSTOR_PERM_WRITE 02
#define MSTOR_PERM_READ 04

#define MSTOR_NID_MAX 0xffffffffffff0000ULL
#define MSTOR_CID_MAX 0xffffffffffff0000ULL
#define MNODE_KEY_LEN (1 + sizeof(uint64_t))
#define MCHILD_KEY_LEN_PREFIX (1 + sizeof(uint64_t))
#define MCHUNK_KEY_LEN (1 + sizeof(uint64_t))
#define MFILE_KEY_LEN (1 + sizeof(uint64_t) + sizeof(uint64_t))
#define MCHILD_KEY_MAX (1 + sizeof(uint64_t) + RF_PCOMP_MAX)
#define MZOMBIE_KEY_LEN (1 + sizeof(uint64_t) + sizeof(uint64_t))

#define MREQ_FLAG_CHECK_PERMS 0x1
#define TMP_CINFO_BUF_SZ 64

/****************************** prototypes ********************************/
static void mstor_leveldb_shutdown(struct mstor *mstor);

/****************************** types ********************************/
/** A metadata node representing either a file or a directory
 */
struct mnode {
	/** Node id */
	uint64_t nid;
	/** Pointer to data record */
	struct mnode_payload *val;
};

// TODO: create separate mfile / mdir structures.  mdir doesn't need length and
// atime.
PACKED(
struct mnode_payload {
	uint64_t mtime;
	uint64_t atime;
	uint64_t length;
	uint32_t uid;
	uint32_t gid;
	uint16_t mode_and_type;
});

struct mstor {
	/** leveldb database */
	leveldb_t *ldb;
	/** leveldb read options */
	leveldb_readoptions_t *lreadopt;
	/** leveldb write options */
	leveldb_writeoptions_t *lwropt;
	/** leveldb LRU cache */
	leveldb_cache_t *lcache;
	/** Next node ID to use */
	uint64_t next_nid;
	/** The minimum number of seconds that we will sequester a file before
	 * deleting it. */
	int min_sequester_time;
	/** Minimum replication level */
	int min_repl;
	/** Mandated replication level */
	int man_repl;
	/** Protects next_nid */
	pthread_mutex_t next_nid_lock;
	/** Next node ID to use */
	uint64_t next_cid;
	/** Protects next_cid */
	pthread_mutex_t next_cid_lock;
	/** user data */
	struct udata *udata;
};

/****************************** functions ********************************/
const char *mstor_op_ty_to_str(enum mstor_op_ty op)
{
	switch (op) {
	case MSTOR_OP_CREAT:
		return "MSTOR_OP_CREAT";
	case MSTOR_OP_OPEN:
		return "MSTOR_OP_OPEN";
	case MSTOR_OP_CHUNKFIND:
		return "MSTOR_OP_CHUNKFIND";
	case MSTOR_OP_CHUNKALLOC:
		return "MSTOR_OP_CHUNKALLOC";
	case MSTOR_OP_MKDIRS:
		return "MSTOR_OP_MKDIRS";
	case MSTOR_OP_LISTDIR:
		return "MSTOR_OP_LISTDIR";
	case MSTOR_OP_STAT:
		return "MSTOR_OP_STAT";
	case MSTOR_OP_CHMOD:
		return "MSTOR_OP_CHMOD";
	case MSTOR_OP_CHOWN:
		return "MSTOR_OP_CHOWN";<commit_msg>mstor: add locking clarification for mstor->udata<commit_after>/*
 * Copyright 2011-2012 the Redfish authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "common/config/mstorc.h"
#include "core/glitch_log.h"
#include "jorm/jorm_const.h"
#include "mds/defaults.h"
#include "mds/limits.h"
#include "mds/mstor.h"
#include "mds/user.h"
#include "msg/generic.h"
#include "util/error.h"
#include "util/fast_log.h"
#include "util/packed.h"
#include "util/path.h"
#include "util/simple_io.h"
#include "util/string.h"

#include <errno.h>
#include <inttypes.h>
#include <leveldb/c.h>
#include <pthread.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

/* leveldb storage scheme:
 * for file and directory nodes:
 *	n[8-byte node-id] => mnode
 * for files:
 *      f[8-byte node-id][8-byte offset] => 8-byte chunk ID
 * for directory children:
 *	c[8-byte node-id][child-name] => 8-byte child ID
 * for chunks:
 *      h[8-byte-chunk-id] => <packed-array of 4-byte OSD-IDs>
 * for zombie chunks:
 *      z[8-byte-death-time][8-byte-zombie-chunk-id] => []
 */
/****************************** constants ********************************/

#define MSTOR_CUR_VERSION 0x000000001U
#define MSTOR_VERSION_MAGIC "Fish"
#define MSTOR_VERSION_MAGIC_LEN 4
#define MSTOR_VERSION_BODY_LEN 8
#define MSTOR_VERSION_INVAL 0xffffffffU

/** TODO: allocate chunk IDs in a per-delegation fashion */
#define MSTOR_INIT_CID 1

#define MSTOR_ROOT_NID 0

#define MSTOR_PERM_EXEC 01
#define MSTOR_PERM_WRITE 02
#define MSTOR_PERM_READ 04

#define MSTOR_NID_MAX 0xffffffffffff0000ULL
#define MSTOR_CID_MAX 0xffffffffffff0000ULL
#define MNODE_KEY_LEN (1 + sizeof(uint64_t))
#define MCHILD_KEY_LEN_PREFIX (1 + sizeof(uint64_t))
#define MCHUNK_KEY_LEN (1 + sizeof(uint64_t))
#define MFILE_KEY_LEN (1 + sizeof(uint64_t) + sizeof(uint64_t))
#define MCHILD_KEY_MAX (1 + sizeof(uint64_t) + RF_PCOMP_MAX)
#define MZOMBIE_KEY_LEN (1 + sizeof(uint64_t) + sizeof(uint64_t))

#define MREQ_FLAG_CHECK_PERMS 0x1
#define TMP_CINFO_BUF_SZ 64

/****************************** prototypes ********************************/
static void mstor_leveldb_shutdown(struct mstor *mstor);

/****************************** types ********************************/
/** A metadata node representing either a file or a directory
 */
struct mnode {
	/** Node id */
	uint64_t nid;
	/** Pointer to data record */
	struct mnode_payload *val;
};

// TODO: create separate mfile / mdir structures.  mdir doesn't need length and
// atime.
PACKED(
struct mnode_payload {
	uint64_t mtime;
	uint64_t atime;
	uint64_t length;
	uint32_t uid;
	uint32_t gid;
	uint16_t mode_and_type;
});

struct mstor {
	/** leveldb database */
	leveldb_t *ldb;
	/** leveldb read options */
	leveldb_readoptions_t *lreadopt;
	/** leveldb write options */
	leveldb_writeoptions_t *lwropt;
	/** leveldb LRU cache */
	leveldb_cache_t *lcache;
	/** Next node ID to use */
	uint64_t next_nid;
	/** The minimum number of seconds that we will sequester a file before
	 * deleting it. */
	int min_sequester_time;
	/** Minimum replication level */
	int min_repl;
	/** Mandated replication level */
	int man_repl;
	/** Protects next_nid */
	pthread_mutex_t next_nid_lock;
	/** Next node ID to use */
	uint64_t next_cid;
	/** Protects next_cid */
	pthread_mutex_t next_cid_lock;
	/** user data.  You cannot modify this without quiescing all threads
	 * that modify the mstor. */
	struct udata *udata;
};

/****************************** functions ********************************/
const char *mstor_op_ty_to_str(enum mstor_op_ty op)
{
	switch (op) {
	case MSTOR_OP_CREAT:
		return "MSTOR_OP_CREAT";
	case MSTOR_OP_OPEN:
		return "MSTOR_OP_OPEN";
	case MSTOR_OP_CHUNKFIND:
		return "MSTOR_OP_CHUNKFIND";
	case MSTOR_OP_CHUNKALLOC:
		return "MSTOR_OP_CHUNKALLOC";
	case MSTOR_OP_MKDIRS:
		return "MSTOR_OP_MKDIRS";
	case MSTOR_OP_LISTDIR:
		return "MSTOR_OP_LISTDIR";
	case MSTOR_OP_STAT:
		return "MSTOR_OP_STAT";
	case MSTOR_OP_CHMOD:
		return "MSTOR_OP_CHMOD";
	case MSTOR_OP_CHOWN:
		return "MSTOR_OP_CHOWN";
{'fraction_non_alphanumeric': 0.15961921629400044, 'fraction_numerical': 0.02147442993137038, 'mean_word_length': 4.810289389067524, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6361758', 'n_tokens_mistral': 3502, 'n_tokens_neox': 3317, 'n_words': 859}