<commit_before>

/*  toysql_lexer.c                                */  
/*  The beginnings of a lexer for "toy SQL".      */ 
/*  This code is released to the public domain.   */ 
/*  "Share and enjoy...."  ;)                     */ 


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h> 
#include <malloc.h> 
#include <stdbool.h> 


char *toks[] = { "select", "from", "where", "and", "or", 
  "not", "in", "=", "<", ">", "<=", ">=", "!=", ",", ";" }; 

/* This enum includes tokens for ints, string literals */ 
/* and variable (column) names. */ 
enum Toktype { TOK_SELECT, TOK_FROM, TOK_WHERE, TOK_AND, 
  TOK_IN, TOK_INT, TOK_STRING, TOK_COLNAME, TOK_EQ, TOK_LT, TOK_GT, 
  TOK_LE, TOK_GE, TOK_NE, TOK_COMMA, TOK_SEMI };    
   
    
/* Types. */ 
/* These are the broad category types of tokens.             */ 
/* They are used to narrow down the token type, like this -  */ 
/*   if type == T_KEYWORD                                    */ 
/* { if tok == "select" ( toktype = TOK_SELECT )             */           
enum Type { T_INT, T_STRING, T_KEYWORD, T_COLNAME, T_SYM }; 

/* Value */ 
typedef struct Value { 
	    enum Type type;           /* Broad category of token. */ 
	    enum Toktype tokentype;   /* The exact token type.    */ 
	    union  { 
			     int integer; 
			     char *string; 
			   } v;               /* The token value. */   
} Value; 


/* Lexer */ 
struct lexer { 
	    FILE *fname; 
	    int colnum; 
	    int linenum; 	   
	    Value *val;    /* As defined above. */ 
}  	    





int main() 
{ 
 
 
 
 
 
 
 
 
 
 
 char *test = "select \"foo\" from test;" ; 
  
 tok(test);     
                                               
 return 0;
 
 
}  

<commit_msg>More work on toysql lexer.<commit_after>

/*  toysql_lexer.c                                */  
/*  The beginnings of a lexer for "toy SQL".      */ 
/*  This code is released to the public domain.   */ 
/*  "Share and enjoy...."  ;)                     */ 


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h> 
#include <malloc.h> 
#include <stdbool.h> 


char *toks[] = { "select", "from", "where", "and", 
  "or", "not", "in", "=", "<", ">", "<=", ">=", "!=", 
  "\"", ",", ";" }; 

/* This enum includes tokens for ints, string literals */ 
/* and variable (column) names. */ 
enum Toktype { TOK_SELECT, TOK_FROM, TOK_WHERE, TOK_AND, 
  TOK_IN, TOK_INT, TOK_STRING, TOK_COLNAME, TOK_EQ, TOK_LT, 
  TOK_GT, TOK_LE, TOK_GE, TOK_NE, TOK_DBLQUOTE, TOK_COMMA, 
  TOK_SEMI, TOK_EOF };    
   
    
/* Types. */ 
/* These are the broad category types of tokens.             */ 
/* They are used to narrow down the token type, like this -  */ 
/*   if type == T_KEYWORD                                    */ 
/* { if tok == "select" ( toktype = TOK_SELECT )             */  
/* Later on, the T_INT and T_STRING types will be used for   */  
/* doing functions.                                          */          
enum Type { T_INT, T_STRING, T_KEYWORD, T_COLNAME, T_SYM }; 

/* Value */ 
typedef struct Value { 
	    enum Type type;           /* Broad category of token. */ 
	    enum Toktype tokentype;   /* The exact token type.    */ 
	    union  { 
			     int integer; 
			     char *string; 
			   } v;               /* The token value. */   
} Value; 


/* Lexer */ 
/* ( Look at including a token-buffer and stream-buffer here. ) */ 
struct lexer { 
	    FILE *fname; 
	    int colnum; 
	    int linenum; 	   
	    Value *val;    /* As defined above. */ 
} ;    	    











int main() 
{ 
 
 
 
 
 
 
 
 
 
 
 char *test = "select \"foo\" from test;" ; 
  
 /* tok(test);   */   
                                               
 return 0;
 
 
}  


{'fraction_non_alphanumeric': 0.14551333872271624, 'fraction_numerical': 0.0005389382915656157, 'mean_word_length': 1.5200271554650373, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4283493', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1223, 'n_words': 347}