<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <ctype.h>

#include "../util/dynarray.h"
#include "../util/alloc.h"
#include "../util/util.h"

#include "tokenise.h"
#include "str.h"

/* start must be free'd if not returned */
token **tokenise(const char *line, int until_close_paren)
{
	token **tokens = NULL;
	const char *p;

	for(p = line; *p; p++){
		token *t = umalloc(sizeof *t);
		char c;

		dynarray_add(&tokens, t);

		while(isspace(*p)){
			t->had_whitespace = 1;
			p++;
		}

		if(!*p){
			t->w = ustrdup("");
			break;
		}

		c = *p;
		if(isalpha(c) || c == '_'){
			const char *start = p;

			t->tok = TOKEN_WORD;

			for(p++; *p; p++)
				if(!(isalnum(*p) || *p == '_'))
					break;

			t->w = ustrdup2(start, p);
			p--;

		}else if(c == ','){
			t->tok = TOKEN_COMMA;
		}else if(c == '('){
			t->tok = TOKEN_OPEN_PAREN;
		}else if(c == ')'){
			t->tok = TOKEN_CLOSE_PAREN;
			if(until_close_paren){
				p++;
				break; /* exit early */
			}
		}else if(!strncmp(p, "...", 3)){
			t->tok = TOKEN_ELIPSIS;
			p += 2;
		}else if(c == '"'){
			char *end  = strchr(p + 1, '"');

			/* guaranteed, since strip_comment() checks */
			while(end[-1] == '\\')
				end = strchr(end + 1, '"');

			t->w = ustrdup2(p, end + 1);
			p = end + 1;

			t->tok = TOKEN_STRING;
		}else{
			t->tok = TOKEN_OTHER;
			t->w = ustrdup2(p, p + 1);
		}
	}

	if(*p){
		token *t = umalloc(sizeof *t);
		dynarray_add(&tokens, t);
		if(isspace(*p))
			t->had_whitespace = 1;
		t->tok = TOKEN_OTHER;
		t->w = ustrdup(p);
	}

	/* trim tokens */
	if(tokens){
		int i;
		for(i = 0; tokens[i]; i++)
			if(tokens[i]->w)
				str_trim(tokens[i]->w);
	}

	return tokens;
}

const char *token_str(token *t)
{
	switch(t->tok){
		case TOKEN_OTHER:
		case TOKEN_WORD:
		case TOKEN_STRING:<commit_msg>Parsing for "#" and "##"<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <ctype.h>

#include "../util/dynarray.h"
#include "../util/alloc.h"
#include "../util/util.h"

#include "tokenise.h"
#include "str.h"

/* start must be free'd if not returned */
token **tokenise(const char *line, int until_close_paren)
{
	token **tokens = NULL;
	const char *p;

	for(p = line; *p; p++){
		token *t = umalloc(sizeof *t);
		char c;

		dynarray_add(&tokens, t);

		while(isspace(*p)){
			t->had_whitespace = 1;
			p++;
		}

		if(!*p){
			t->w = ustrdup("");
			break;
		}

		c = *p;
		if(isalpha(c) || c == '_'){
			const char *start = p;

			t->tok = TOKEN_WORD;

			for(p++; *p; p++)
				if(!(isalnum(*p) || *p == '_'))
					break;

			t->w = ustrdup2(start, p);
			p--;

		}else switch(c){
			case ',':
				t->tok = TOKEN_COMMA;
				break;
			case '(':
				t->tok = TOKEN_OPEN_PAREN;
				break;
			case ')':
				t->tok = TOKEN_CLOSE_PAREN;
				if(until_close_paren){
					p++;
					break; /* exit early */
				}
				break;
			case '"':
			{
				char *end  = strchr(p + 1, '"');

				/* guaranteed, since strip_comment() checks */
				while(end[-1] == '\\')
					end = strchr(end + 1, '"');

				t->w = ustrdup2(p, end + 1);
				p = end + 1;

				t->tok = TOKEN_STRING;
				break;
			}
			case '#':
				if(p[1] == '#')
					t->tok = TOKEN_HASH_JOIN, p++;
				else
					t->tok = TOKEN_HASH_QUOTE;
				break;
			default:
				if(!strncmp(p, "...", 3)){
					t->tok = TOKEN_ELIPSIS;
					p += 2;
				}else{
					t->tok = TOKEN_OTHER;
					t->w = ustrdup2(p, p + 1);
				}
		}
	}

	if(*p){
		token *t = umalloc(sizeof *t);
		dynarray_add(&tokens, t);
		if(isspace(*p))
			t->had_whitespace = 1;
		t->tok = TOKEN_OTHER;
		t->w = ustrdup(p);
	}

	/* trim tokens */
	if(tokens){
		int i;
		for(i = 0; tokens[i]; i++)
			if(tokens[i]->w)
				str_trim(tokens[i]->w);
	}

	return tokens;
}

const char *token_str(token *t)
{
	switch(t->tok){
		case TOKEN_OTHER:
		case TOKEN_WORD:
		case TOKEN_STRING:
{'fraction_non_alphanumeric': 0.21804511278195488, 'fraction_numerical': 0.007518796992481203, 'mean_word_length': 2.6123595505617976, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7617454', 'n_tokens_mistral': 2020, 'n_tokens_neox': 1668, 'n_words': 355}