<commit_before>/*
 * token.h
 *
 * The muforth compiler was replaced with an interpreter in
 * July of 2004.  The goal of this transition was to create
 * a muforth environment that could run on all processor
 * families.  Then, one would build, in forth, a meta-compiler.
 * The interpreter is builds a stream of simple byte-sized
 * opcodes with, in some cases, in-lined data.
 *
 * The opcodes supported are:
 *
 * 00 - NOP
 *
 * 10 - CALL (Target address follows instruction)
 * 11 - JUMP (Target address follows instruction)
 * 12 - Execute (Target address is on the data stack)
 * 13 - RET (Return from CALL)
 *
 * 20 - Push Literal (data follows instruction)
 * 21 - Load Literal (into a holding buffer, data follows instruction)
 * 22 - Push Held Literal (from the holding buffer)
 *
 * 30 - Unconditional Branch
 * 31 - Destructive Branch If Zero, drop top element
 * 32 - Non-destructive Branch If Zero, do not modify stack
 * 
 * 40 - For
 * 41 - Next
 * 42 - I (push current for loop index)
 * 43 - J (push outer loop index within same word)
 *
 * 60 - Push to R stack
 * 61 - Pop from R stack
 * 62 - Copy from R stack
 *
 * Note: The R stack is -not- the host environments return
 * stack.  It is a stack different from the data stack.
 */
#ifndef __MUFORTH_TOKEN_H__
#define __MUFORTH_TOKEN_H__

typedef enum opcode_e opcode_t;
enum opcode_e {
	NOP,

	CALL = 0x10,
	JUMP,
	EXEC,
	RET,

	LITERAL = 0x20,
	LIT_LOAD,
	LIT_PUSH,

	BRANCH = 0x30,
	ZBRANCH,
	_ZBRANCH,

	QFOR = 0x40,
	NEXT,
	I,
	J,

	DROP = 0x60,
	DROP2,
	PUSH_TO_R,
	PUSH2_TO_R,
	POP_FROM_R,
	POP2_FROM_R,
	COPY_FROM_R,
	SHUNT,
};

#endif /* __MUFORTH_TOKEN_H__ */
<commit_msg>Added a BRK opcode and removed the hard coded numbers for the opcodes. BRK will be used with the debugger.  The hard coded numbers was dumb. The instruction opcodes are not written to disk or passed across the wire so the opcode numbers is purely ephemeral, they last only as long as the image is running.  From build to build they could be different with no impact the the correctness of the code.  That's why I believe the hard coded numbers was dumb.<commit_after>/*
 * token.h
 *
 * The muforth compiler was replaced with an interpreter in
 * July of 2004.  The goal of this transition was to create
 * a muforth environment that could run on all processor
 * families.  Then, one would build, in forth, a meta-compiler.
 * The interpreter is builds a stream of simple byte-sized
 * opcodes with, in some cases, in-lined data.
 *
 * The opcodes supported are:
 *
 * NOP
 *
 * CALL (Target address follows instruction)
 * JUMP (Target address follows instruction)
 * Execute (Target address is on the data stack)
 * RET (Return from CALL)
 *
 * Push Literal (data follows instruction)
 * Load Literal (into a holding buffer, data follows instruction)
 * Push Held Literal (from the holding buffer)
 *
 * Unconditional Branch
 * Destructive Branch If Zero, drop top element
 * Non-destructive Branch If Zero, do not modify stack
 * 
 * For
 * Next
 * I (push current for loop index)
 * J (push outer loop index within same word)
 *
 * Push to R stack
 * Pop from R stack
 * Copy from R stack
 *
 * Note: The R stack is -not- the host environments return
 * stack.  It is a stack different from the data stack.
 */
#ifndef __MUFORTH_TOKEN_H__
#define __MUFORTH_TOKEN_H__

typedef enum opcode_e opcode_t;
enum opcode_e {
	BRK,
	NOP,

	JUMP,
	CALL,
	EXEC,
	RET,

	LITERAL,
	LIT_LOAD,
	LIT_PUSH,

	BRANCH,
	ZBRANCH,
	_ZBRANCH,

	QFOR,
	NEXT,
	I,
	J,

	DROP,
	DROP2,
	PUSH_TO_R,
	PUSH2_TO_R,
	POP_FROM_R,
	POP2_FROM_R,
	COPY_FROM_R,
	SHUNT,
};

#endif /* __MUFORTH_TOKEN_H__ */

{'fraction_non_alphanumeric': 0.0744006613392119, 'fraction_numerical': 0.017911270322402865, 'mean_word_length': 3.3840579710144927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6549214', 'n_tokens_mistral': 1321, 'n_tokens_neox': 1214, 'n_words': 514}