<commit_before>/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."

/* This file defines the logformat in an executable fashion.
 * This code is used to generate
 *   The code that writes into the log.
 *   The code that reads the log and prints it to stdout (the log_print utility)
 *   The code that reads the log for recovery.
 *   The struct definitions.
 *   The Latex documentation.
 */
#include "toku_assert.h"
#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

typedef struct field {
    char *type;
    char *name;
    char *format; // optional format string
} F;

#define NULLFIELD {0,0,0}
#define FA (F[])

struct logtype {
    char *name;
    unsigned int command_and_flags;
    struct field *fields;
};

#define GEN_ROLLBACK (1<<9)

// In the fields, don't mention the command, the LSN, the CRC or the trailing LEN.

int logformat_version_number = 0;

const struct logtype logtypes[] = {
    {"commit", 'C', FA{{"TXNID", "txnid", 0},NULLFIELD}},
    {"delete", 'D', FA{{"FILENUM", "filenum", 0},
		       {"DISKOFF", "diskoff", 0},
		       {"BYTESTRING", "key", 0},
		       {"BYTESTRING", "data", 0},
		       NULLFIELD}},
    {"fcreate", 'F'+GEN_ROLLBACK, FA{{"TXNID",      "txnid", 0},
				     {"BYTESTRING", "fname", 0},
				     {"u_int32_t",  "mode",  "0%o"},
				     NULLFIELD}},
    {"fheader", 'H',  FA{{"TXNID",      "txnid", 0},
			 {"FILENUM",    "filenum", 0},
			 {"LOGGEDBRTHEADER",  "header", 0},
			 NULLFIELD}},
    {"newbrtnode", 'N', FA{{"FILENUM", "filenum", 0},
			   {"DISKOFF", "diskoff", 0},
			   {"u_int32_t", "height", 0},
			   {"u_int32_t", "nodesize", 0},
			   {"u_int8_t", "is_dup_sort", 0},
			   {"u_int32_t", "rand4fingerprint", "%08x"},
			   NULLFIELD}},
    {"changechildfingerprint", 'f', FA{{"FILENUM", "filenum", 0},
				       {"DISKOFF", "diskoff", 0},
				       {"u_int32_t", "childnum", 0},
				       {"u_int32_t", "oldfingerprint", "%08x"},
				       {"u_int32_t", "newfingerprint", "%08x"},
				       NULLFIELD}},
    {"changeunnamedroot", 'u', FA{{"FILENUM", "filenum", 0},
				  {"DISKOFF", "oldroot", 0},
				  {"DISKOFF", "newroot", 0},
				  NULLFIELD}},
    {"changenamedroot", 'n', FA{{"FILENUM", "filenum", 0},
				{"BYTESTRING", "name", 0},
				{"DISKOFF", "oldroot", 0},
				{"DISKOFF", "newroot", 0},
				NULLFIELD}},
    {"changeunusedmemory", 'm',  FA{{"FILENUM", "filenum", 0},
				    {"DISKOFF", "oldunused", 0},
				    {"DISKOFF", "newunused", 0},
				    NULLFIELD}},
    {"addchild", 'c', FA{{"FILENUM", "filenum", 0},
			 {"DISKOFF", "diskoff", 0},
			 {"u_int32_t", "childnum", 0}, // children scoot over
			 {"DISKOFF",   "child", 0},
			 {"u_int32_t", "childfingerprint", "%08x"},
			 NULLFIELD}},
    {"delchild", 'r', FA{{"FILENUM", "filenum", 0},
			 {"DISKOFF", "diskoff", 0},
			 {"u_int32_t", "childnum", 0}, // children scoot over
			 {"DISKOFF",   "child", 0},
			 {"u_int32_t", "childfingerprint", "%08x"},
			 {"BYTESTRING", "pivotkey", 0},
			 NULLFIELD}},
    {"setchild", 'i', FA{{"FILENUM",   "filenum", 0},
			 {"DISKOFF",   "diskoff", 0},
			 {"u_int32_t", "childnum", 0},
			 {"DISKOFF",   "oldchild", 0},
			 {"DISKOFF",   "newchild", 0},
			 NULLFIELD}},
    {"setpivot", 'k', FA{{"FILENUM", "filenum", 0},
			 {"DISKOFF", "diskoff", 0},
			 {"u_int32_t", "childnum", 0},
			 {"BYTESTRING", "pivotkey", 0},
			 NULLFIELD}},
    {"fopen",   'O', FA{{"TXNID",      "txnid", 0},
			{"BYTESTRING", "fname", 0},
			{"FILENUM",    "filenum", 0},
			NULLFIELD}},
    {"brtdeq",       'U', FA{{"FILENUM",    "filenum", 0},
			     {"DISKOFF",    "diskoff", 0},
			     {"u_int32_t",  "childnum", 0},
			     {"TXNID",      "xid", 0},
			     {"u_int32_t",  "typ", 0},
			     {"BYTESTRING", "key", 0},
			     {"BYTESTRING", "data", 0},
			     {"u_int32_t",  "oldfingerprint", "%08x"},
			     {"u_int32_t",  "newfingerprint", "%08x"},
			     NULLFIELD}},
    {"brtenq",       'Q', FA{{"FILENUM",    "filenum", 0},
			     {"DISKOFF",    "diskoff", 0},
			     {"u_int32_t",  "childnum", 0},
			     {"TXNID",      "xid", 0},
			     {"u_int32_t",  "typ", 0},
			     {"BYTESTRING", "key", 0},
			     {"BYTESTRING", "data", 0},
			     {"u_int32_t",  "oldfingerprint", "%08x"},
			     {"u_int32_t",  "newfingerprint", "%08x"},
			     NULLFIELD}},
    {"insertinleaf", 'I', FA{{"TXNID",      "txnid", 0},
			     {"FILENUM",    "filenum", 0},
			     {"DISKOFF",    "diskoff", 0},
			     {"u_int32_t",  "pmaidx", 0},
			     {"BYTESTRING", "key", 0},
			     {"BYTESTRING", "data", 0},
			     NULLFIELD}},
    {"tl_delete",    'K'GEN_ROLLBACK, FA{{"FILENUM", "filenum", 0}, // Note a delete for rollback.  
					 {"BYTESTRING", "key", 0},
					 {"BYTESTRING", "data", 0},
					 NULLFIELD}},
    {"deleteinleaf", 'd', FA{{"TXNID",      "txnid", 0},
			     {"FILENUM",    "filenum", 0},
			     {"DISKOFF",    "diskoff", 0},
			     {"u_int32_t",  "pmaidx", 0},
			     {"BYTESTRING", "key", 0},
			     {"BYTESTRING", "data", 0},
                             NULLFIELD}},
    {"resizepma", 'R', FA{{"FILENUM",    "filenum", 0},
			  {"DISKOFF",    "diskoff", 0},
			  {"u_int32_t",  "oldsize", 0},
			  {"u_int32_t",  "newsize", 0},
			  NULLFIELD}},
    {"pmadistribute", 'M', FA{{"FILENUM",    "filenum", 0},
			      {"DISKOFF",    "old_diskoff", 0},
			      {"DISKOFF",    "new_diskoff", 0},
			      {"INTPAIRARRAY",   "fromto",    0},
			      NULLFIELD}},
    {0,0,FA{NULLFIELD}}
};
    
#define DO_LOGTYPES(lt, body) ({ \
    const struct logtype *lt;    \
    for (lt=&logtypes[0]; lt->name; lt++) { \
	body; \
    } })
<commit_msg>Typo. Addresses #27.<commit_after>/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."

/* This file defines the logformat in an executable fashion.
 * This code is used to generate
 *   The code that writes into the log.
 *   The code that reads the log and prints it to stdout (the log_print utility)
 *   The code that reads the log for recovery.
 *   The struct definitions.
 *   The Latex documentation.
 */
#include "toku_assert.h"
#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

typedef struct field {
    char *type;
    char *name;
    char *format; // optional format string
} F;

#define NULLFIELD {0,0,0}
#define FA (F[])

struct logtype {
    char *name;
    unsigned int command_and_flags;
    struct field *fields;
};

#define GEN_ROLLBACK (1<<9)

// In the fields, don't mention the command, the LSN, the CRC or the trailing LEN.

int logformat_version_number = 0;

const struct logtype logtypes[] = {
    {"commit", 'C', FA{{"TXNID", "txnid", 0},NULLFIELD}},
    {"delete", 'D', FA{{"FILENUM", "filenum", 0},
		       {"DISKOFF", "diskoff", 0},
		       {"BYTESTRING", "key", 0},
		       {"BYTESTRING", "data", 0},
		       NULLFIELD}},
    {"fcreate", 'F'+GEN_ROLLBACK, FA{{"TXNID",      "txnid", 0},
				     {"BYTESTRING", "fname", 0},
				     {"u_int32_t",  "mode",  "0%o"},
				     NULLFIELD}},
    {"fheader", 'H',  FA{{"TXNID",      "txnid", 0},
			 {"FILENUM",    "filenum", 0},
			 {"LOGGEDBRTHEADER",  "header", 0},
			 NULLFIELD}},
    {"newbrtnode", 'N', FA{{"FILENUM", "filenum", 0},
			   {"DISKOFF", "diskoff", 0},
			   {"u_int32_t", "height", 0},
			   {"u_int32_t", "nodesize", 0},
			   {"u_int8_t", "is_dup_sort", 0},
			   {"u_int32_t", "rand4fingerprint", "%08x"},
			   NULLFIELD}},
    {"changechildfingerprint", 'f', FA{{"FILENUM", "filenum", 0},
				       {"DISKOFF", "diskoff", 0},
				       {"u_int32_t", "childnum", 0},
				       {"u_int32_t", "oldfingerprint", "%08x"},
				       {"u_int32_t", "newfingerprint", "%08x"},
				       NULLFIELD}},
    {"changeunnamedroot", 'u', FA{{"FILENUM", "filenum", 0},
				  {"DISKOFF", "oldroot", 0},
				  {"DISKOFF", "newroot", 0},
				  NULLFIELD}},
    {"changenamedroot", 'n', FA{{"FILENUM", "filenum", 0},
				{"BYTESTRING", "name", 0},
				{"DISKOFF", "oldroot", 0},
				{"DISKOFF", "newroot", 0},
				NULLFIELD}},
    {"changeunusedmemory", 'm',  FA{{"FILENUM", "filenum", 0},
				    {"DISKOFF", "oldunused", 0},
				    {"DISKOFF", "newunused", 0},
				    NULLFIELD}},
    {"addchild", 'c', FA{{"FILENUM", "filenum", 0},
			 {"DISKOFF", "diskoff", 0},
			 {"u_int32_t", "childnum", 0}, // children scoot over
			 {"DISKOFF",   "child", 0},
			 {"u_int32_t", "childfingerprint", "%08x"},
			 NULLFIELD}},
    {"delchild", 'r', FA{{"FILENUM", "filenum", 0},
			 {"DISKOFF", "diskoff", 0},
			 {"u_int32_t", "childnum", 0}, // children scoot over
			 {"DISKOFF",   "child", 0},
			 {"u_int32_t", "childfingerprint", "%08x"},
			 {"BYTESTRING", "pivotkey", 0},
			 NULLFIELD}},
    {"setchild", 'i', FA{{"FILENUM",   "filenum", 0},
			 {"DISKOFF",   "diskoff", 0},
			 {"u_int32_t", "childnum", 0},
			 {"DISKOFF",   "oldchild", 0},
			 {"DISKOFF",   "newchild", 0},
			 NULLFIELD}},
    {"setpivot", 'k', FA{{"FILENUM", "filenum", 0},
			 {"DISKOFF", "diskoff", 0},
			 {"u_int32_t", "childnum", 0},
			 {"BYTESTRING", "pivotkey", 0},
			 NULLFIELD}},
    {"fopen",   'O', FA{{"TXNID",      "txnid", 0},
			{"BYTESTRING", "fname", 0},
			{"FILENUM",    "filenum", 0},
			NULLFIELD}},
    {"brtdeq",       'U', FA{{"FILENUM",    "filenum", 0},
			     {"DISKOFF",    "diskoff", 0},
			     {"u_int32_t",  "childnum", 0},
			     {"TXNID",      "xid", 0},
			     {"u_int32_t",  "typ", 0},
			     {"BYTESTRING", "key", 0},
			     {"BYTESTRING", "data", 0},
			     {"u_int32_t",  "oldfingerprint", "%08x"},
			     {"u_int32_t",  "newfingerprint", "%08x"},
			     NULLFIELD}},
    {"brtenq",       'Q', FA{{"FILENUM",    "filenum", 0},
			     {"DISKOFF",    "diskoff", 0},
			     {"u_int32_t",  "childnum", 0},
			     {"TXNID",      "xid", 0},
			     {"u_int32_t",  "typ", 0},
			     {"BYTESTRING", "key", 0},
			     {"BYTESTRING", "data", 0},
			     {"u_int32_t",  "oldfingerprint", "%08x"},
			     {"u_int32_t",  "newfingerprint", "%08x"},
			     NULLFIELD}},
    {"insertinleaf", 'I', FA{{"TXNID",      "txnid", 0},
			     {"FILENUM",    "filenum", 0},
			     {"DISKOFF",    "diskoff", 0},
			     {"u_int32_t",  "pmaidx", 0},
			     {"BYTESTRING", "key", 0},
			     {"BYTESTRING", "data", 0},
			     NULLFIELD}},
    {"tl_delete",    'K'+GEN_ROLLBACK, FA{{"FILENUM", "filenum", 0}, // Note a delete for rollback.  
					  {"BYTESTRING", "key", 0},
					  {"BYTESTRING", "data", 0},
					  NULLFIELD}},
    {"deleteinleaf", 'd', FA{{"TXNID",      "txnid", 0},
			     {"FILENUM",    "filenum", 0},
			     {"DISKOFF",    "diskoff", 0},
			     {"u_int32_t",  "pmaidx", 0},
			     {"BYTESTRING", "key", 0},
			     {"BYTESTRING", "data", 0},
                             NULLFIELD}},
    {"resizepma", 'R', FA{{"FILENUM",    "filenum", 0},
			  {"DISKOFF",    "diskoff", 0},
			  {"u_int32_t",  "oldsize", 0},
			  {"u_int32_t",  "newsize", 0},
			  NULLFIELD}},
    {"pmadistribute", 'M', FA{{"FILENUM",    "filenum", 0},
			      {"DISKOFF",    "old_diskoff", 0},
			      {"DISKOFF",    "new_diskoff", 0},
			      {"INTPAIRARRAY",   "fromto",    0},
			      NULLFIELD}},
    {0,0,FA{NULLFIELD}}
};
    
#define DO_LOGTYPES(lt, body) ({ \
    const struct logtype *lt;    \
    for (lt=&logtypes[0]; lt->name; lt++) { \
	body; \
    } })

{'fraction_non_alphanumeric': 0.22697512003491924, 'fraction_numerical': 0.03072893932780445, 'mean_word_length': 2.7389033942558747, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6842859', 'n_tokens_mistral': 5127, 'n_tokens_neox': 4287, 'n_words': 1024}