<commit_before>/*	____________________________________________________________________________

	                 Scanner Component for the Micro A Compiler

	                                mscan.h

	                              Version 2007 - 2016
 
	                           <NAME>
	                     Last Update: August 28, 2007
	                      	   <NAME>
	                     Last Update: January 31, 2016

	The routines in this unit are based on those provided in the book 
	"Crafting A Compiler" by <NAME> and <NAME>, Jr., 
	Benjamin Cummings Publishing Co. (1991).

	See Section 2.2, pp. 25-29.
	____________________________________________________________________________
*/

#ifndef SCANNER
#define SCANNER
#include <string>

const int ID_STRING_LEN = 32;

enum Token  { EOF_SYM, BOOL_SYM, BREAK_SYM, CASE_SYM, CHEESE_SYM,
		DECS_SYM, DO_SYM, ELSE_SYM, END_SYM, FALSE_SYM,
		FLOAT_SYM, FOR_SYM, HIPHIP_SYM, IF_SYM, INT_SYM,
		LISTEN_SYM, OTHERWISE_SYM, UNTIL_SYM, SELECT_SYM, SHOUT_SYM,
		THEN_SYM, TRUE_SYM, WHILE_SYM, LSTAPLE, RSTAPLE,
		LBANANA, RBANANA, LMUSTACHE, RMUSTACHE, COLON,
		SEMICOLON, COMMA, ASSIGN_OP, PLUS_OP, MINUS_OP,
		MULT_OP, DIV_OP, LT_OP, LE_OP, GT_OP, GE_OP,
		EQ_OP1, EQ_OP2, NE_OP, ID, BOOL_LIT, INT_LIT,
		FLOAT_LIT, CHEESE_LIT };

class Scanner
{
public:

	string	tokenBuffer, // current token
			lineBuffer;  // current line
	int		lineNumber;  // line counter

	Scanner();
	// Initializes the scanner object.

	Token GetNextToken();
	// Retrieves the next token available in the input stream.

private:

	void BufferChar(char c);
   // Appends character c to the end of the token buffer provided that the
   // buffer is not full.<commit_msg>Increase the size of the token buffer.<commit_after>/*	____________________________________________________________________________

	                 Scanner Component for the Micro A Compiler

	                                mscan.h

	                              Version 2007 - 2016
 
	                           <NAME>
	                     Last Update: August 28, 2007
	                      	   <NAME>
	                     Last Update: January 31, 2016

	The routines in this unit are based on those provided in the book 
	"Crafting A Compiler" by <NAME> and <NAME>, Jr., 
	<NAME> Publishing Co. (1991).

	See Section 2.2, pp. 25-29.
	____________________________________________________________________________
*/

#ifndef SCANNER
#define SCANNER
#include <string>

const int ID_STRING_LEN = 1024; /* FIXME CHECK FOR MAX SIZE WHEN SCANNING */

enum Token  { EOF_SYM, BOOL_SYM, BREAK_SYM, CASE_SYM, CHEESE_SYM,
		DECS_SYM, DO_SYM, ELSE_SYM, END_SYM, FALSE_SYM,
		FLOAT_SYM, FOR_SYM, HIPHIP_SYM, IF_SYM, INT_SYM,
		LISTEN_SYM, OTHERWISE_SYM, UNTIL_SYM, SELECT_SYM, SHOUT_SYM,
		THEN_SYM, TRUE_SYM, WHILE_SYM, LSTAPLE, RSTAPLE,
		LBANANA, RBANANA, LMUSTACHE, RMUSTACHE, COLON,
		SEMICOLON, COMMA, ASSIGN_OP, PLUS_OP, MINUS_OP,
		MULT_OP, DIV_OP, LT_OP, LE_OP, GT_OP, GE_OP,
		EQ_OP1, EQ_OP2, NE_OP, ID, BOOL_LIT, INT_LIT,
		FLOAT_LIT, CHEESE_LIT };

class Scanner
{
public:

	string	tokenBuffer, // current token
			lineBuffer;  // current line
	int		lineNumber;  // line counter

	Scanner();
	// Initializes the scanner object.

	Token GetNextToken();
	// Retrieves the next token available in the input stream.

private:

	void BufferChar(char c);
   // Appends character c to the end of the token buffer provided that the
   // buffer is not full.
{'fraction_non_alphanumeric': 0.07442803359397625, 'fraction_numerical': 0.020272227048942947, 'mean_word_length': 2.7543478260869567, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3148327', 'n_tokens_mistral': 1290, 'n_tokens_neox': 1115, 'n_words': 345}