<commit_before>#include "pbc.h"
#include "alloc.h"
#include "context.h"
#include "proto.h"
#include "varint.h"

#include <assert.h>

static const char * TYPENAME[] = {
	"invalid",	// 0
	"integer",	// 1
	"real",	// 2
	"boolean",	// 3
	"enum",	// 4
	"string",	// 5
	"message",	// 6
	"fixed64",	// 7
	"fixed32",	// 8
	"bytes",	// 9
	"int64",	// 10
	"uint",	// 11
};

static int
call_unknown(pbc_decoder f, void * ud, int id, struct atom *a, uint8_t * start) {
	union pbc_value v;
	switch (a->wire_id) {
	case WT_VARINT:
		v.i.low = a->v.i.low;
		v.i.hi = a->v.i.hi;
		f(ud, PBC_INT, TYPENAME[PBC_INT], &v, id , NULL);
		break;
	case WT_BIT64:
		v.i.low = a->v.i.low;
		v.i.hi = a->v.i.hi;
		f(ud, PBC_FIXED64, TYPENAME[PBC_FIXED64], &v, id , NULL);
		break;
	case WT_LEND:
		v.s.buffer = (char*)start + a->v.s.start;
		v.s.len = a->v.s.end - a->v.s.start;
		f(ud, PBC_BYTES, TYPENAME[PBC_BYTES], &v, id , NULL);
		break;
	case WT_BIT32:
		v.i.low = a->v.i.low;
		v.i.hi = 0;
		f(ud, PBC_FIXED32, TYPENAME[PBC_FIXED32], &v, id , NULL);
		break;
	default:
		return 1;
	}
	return 0;
}
<commit_msg>bugfix: check unknown field wire type<commit_after>#include "pbc.h"
#include "alloc.h"
#include "context.h"
#include "proto.h"
#include "varint.h"

#include <assert.h>

static const char * TYPENAME[] = {
	"invalid",	// 0
	"integer",	// 1
	"real",	// 2
	"boolean",	// 3
	"enum",	// 4
	"string",	// 5
	"message",	// 6
	"fixed64",	// 7
	"fixed32",	// 8
	"bytes",	// 9
	"int64",	// 10
	"uint",	// 11
};

static int
call_unknown(pbc_decoder f, void * ud, int id, struct atom *a, uint8_t * start) {
	union pbc_value v;
	switch (a->wire_id & 7) {
	case WT_VARINT:
		v.i.low = a->v.i.low;
		v.i.hi = a->v.i.hi;
		f(ud, PBC_INT, TYPENAME[PBC_INT], &v, id , NULL);
		break;
	case WT_BIT64:
		v.i.low = a->v.i.low;
		v.i.hi = a->v.i.hi;
		f(ud, PBC_FIXED64, TYPENAME[PBC_FIXED64], &v, id , NULL);
		break;
	case WT_LEND:
		v.s.buffer = (char*)start + a->v.s.start;
		v.s.len = a->v.s.end - a->v.s.start;
		f(ud, PBC_BYTES, TYPENAME[PBC_BYTES], &v, id , NULL);
		break;
	case WT_BIT32:
		v.i.low = a->v.i.low;
		v.i.hi = 0;
		f(ud, PBC_FIXED32, TYPENAME[PBC_FIXED32], &v, id , NULL);
		break;
	default:
		return 1;
	}
	return 0;
}

{'fraction_non_alphanumeric': 0.21447963800904976, 'fraction_numerical': 0.0330316742081448, 'mean_word_length': 2.85191637630662, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5848105', 'n_tokens_mistral': 1211, 'n_tokens_neox': 1093, 'n_words': 240}