<commit_before>#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <stdarg.h>
#include <stdlib.h>

#include "../util/util.h"
#include "../util/alloc.h"
#include "data_structs.h"
#include "str.h"

int escape_char(int c)
{
	struct
	{
		char from, to;
	} escapechars[] = {
		{ 'n', '\n'  },
		{ 't', '\t'  },
		{ 'b', '\b'  },
		{ 'r', '\r'  },
		{ 'v', '\v'  },
		{ 'f', '\f'  },
		{ '0', '\0'  },
		{ 'e', '\33' },
		{ '\\', '\\' },
		{ '\'', '\'' },
		{ '"',  '"'  }
	};
	unsigned int i;

	for(i = 0; i < sizeof(escapechars) / sizeof(escapechars[0]); i++)
		if(escapechars[i].from == c)
			return escapechars[i].to;

	return -1;
}

void char_seq_to_iv(char *s, intval *iv, int *plen, enum base mode)
{
#define READ_NUM(test, base)             \
			do{                                \
				if(!test)                        \
					break;                         \
				lval = base * lval + *s - '0';   \
				s++;                             \
				while(*s == '_')                 \
					s++;                           \
			}while(1)

	char *const start = s;
	long lval = 0;

	switch(mode){
		case BIN:
			READ_NUM(*s == '0' || *s == '1', 2);
			break;

		case DEC:
			READ_NUM(isdigit(*s), 10);
			break;

		case OCT:
			READ_NUM(isoct(*s), 010);
			break;

		case HEX:
		{
			int charsread = 0;
			do{
				if(isxdigit(*s)){
					lval = 0x10 * lval + (isdigit(tolower(*s)) ? *s - '0' : 10 + tolower(*s) - 'a');
					s++;
				}else{
					break;
				}
				charsread++;
				while(*s == '_')
					s++;
			}while(1);

			if(charsread < 1)
				DIE_AT(NULL, "invalid hex char (read 0 chars, at \"%s\")", s);
			break;
		}
	}

	iv->val = lval;
	*plen = s - start;
}

int escape_multi_char(char *pos, char *pval, int *len)
{
	/* either \x or \[1-7][0-7]* */
	int is_oct;

	if((is_oct = ('0' <= *pos && *pos <= '7')) || *pos == 'x'){
		/* can't do '\b' - already defined */
		intval iv;

		if(is_oct){
			/*
			 * special case: '\0' is not oct
			 */
			if(*pos == '0' && !isoct(pos[1]))
				return 0;

			/* else got an oct, continuez vous */
		}else{
			pos++; /* skip the 'x' */
		}

		char_seq_to_iv(pos, &iv, len, is_oct ? OCT : HEX);

		*pval = iv.val;

		if(!is_oct)
			++*len; /* we stepped over the 'x' */

		return 1;
	}

	return 0;
}

void escape_string(char *old_str, int *plen)
{
	const int old_len = *plen;
	char *new_str = umalloc(*plen);
	int new_i, i;

	/* "parse" into another string */

	for(i = new_i = 0; i < old_len; i++){
		char add;

		if(old_str[i] == '\\'){
			int len;

			i++;

			if(escape_multi_char(old_str + i, &add, &len)){
				i += len - 1;
			}else{
				add = escape_char(old_str[i]);

				if(add == -1)
					DIE_AT(NULL, "unknown escape char '\\%c'", add);
			}
		}else{
			add = old_str[i];
		}

		new_str[new_i++] = add;
	}

	memcpy(old_str, new_str, new_i);
	*plen = new_i;
	free(new_str);
}

int literal_print(FILE *f, const char *s, int len)
{
	for(; len; s++, len--)
		if(*s == '\\'){
			if(fputs("\\\\", f) == EOF)
				return EOF;
		}else if(!isprint(*s)){
			if(fprintf(f, "\\%03o", *s) < 0)
				return EOF;
		}else if(fputc(*s, f) == EOF){
			return EOF;
		}

	return 0;
}
<commit_msg>Fixed parsing of 0b[01]+ constants<commit_after>#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <stdarg.h>
#include <stdlib.h>

#include "../util/util.h"
#include "../util/alloc.h"
#include "data_structs.h"
#include "str.h"

int escape_char(int c)
{
	struct
	{
		char from, to;
	} escapechars[] = {
		{ 'n', '\n'  },
		{ 't', '\t'  },
		{ 'b', '\b'  },
		{ 'r', '\r'  },
		{ 'v', '\v'  },
		{ 'f', '\f'  },
		{ '0', '\0'  },
		{ 'e', '\33' },
		{ '\\', '\\' },
		{ '\'', '\'' },
		{ '"',  '"'  }
	};
	unsigned int i;

	for(i = 0; i < sizeof(escapechars) / sizeof(escapechars[0]); i++)
		if(escapechars[i].from == c)
			return escapechars[i].to;

	return -1;
}

void char_seq_to_iv(char *s, intval *iv, int *plen, enum base mode)
{
#define READ_NUM(test, base)             \
			do{                                \
				if(!(test))                      \
					break;                         \
				lval = base * lval + *s - '0';   \
				s++;                             \
				while(*s == '_')                 \
					s++;                           \
			}while(1)

	char *const start = s;
	long lval = 0;

	switch(mode){
		case BIN:
			READ_NUM(*s == '0' || *s == '1', 2);
			break;

		case DEC:
			READ_NUM(isdigit(*s), 10);
			break;

		case OCT:
			READ_NUM(isoct(*s), 010);
			break;

		case HEX:
		{
			int charsread = 0;
			do{
				if(isxdigit(*s)){
					lval = 0x10 * lval + (isdigit(tolower(*s)) ? *s - '0' : 10 + tolower(*s) - 'a');
					s++;
				}else{
					break;
				}
				charsread++;
				while(*s == '_')
					s++;
			}while(1);

			if(charsread < 1)
				DIE_AT(NULL, "invalid hex char (read 0 chars, at \"%s\")", s);
			break;
		}
	}

	iv->val = lval;
	*plen = s - start;
}

int escape_multi_char(char *pos, char *pval, int *len)
{
	/* either \x or \[1-7][0-7]* */
	int is_oct;

	if((is_oct = ('0' <= *pos && *pos <= '7')) || *pos == 'x'){
		/* can't do '\b' - already defined */
		intval iv;

		if(is_oct){
			/*
			 * special case: '\0' is not oct
			 */
			if(*pos == '0' && !isoct(pos[1]))
				return 0;

			/* else got an oct, continuez vous */
		}else{
			pos++; /* skip the 'x' */
		}

		char_seq_to_iv(pos, &iv, len, is_oct ? OCT : HEX);

		*pval = iv.val;

		if(!is_oct)
			++*len; /* we stepped over the 'x' */

		return 1;
	}

	return 0;
}

void escape_string(char *old_str, int *plen)
{
	const int old_len = *plen;
	char *new_str = umalloc(*plen);
	int new_i, i;

	/* "parse" into another string */

	for(i = new_i = 0; i < old_len; i++){
		char add;

		if(old_str[i] == '\\'){
			int len;

			i++;

			if(escape_multi_char(old_str + i, &add, &len)){
				i += len - 1;
			}else{
				add = escape_char(old_str[i]);

				if(add == -1)
					DIE_AT(NULL, "unknown escape char '\\%c'", add);
			}
		}else{
			add = old_str[i];
		}

		new_str[new_i++] = add;
	}

	memcpy(old_str, new_str, new_i);
	*plen = new_i;
	free(new_str);
}

int literal_print(FILE *f, const char *s, int len)
{
	for(; len; s++, len--)
		if(*s == '\\'){
			if(fputs("\\\\", f) == EOF)
				return EOF;
		}else if(!isprint(*s)){
			if(fprintf(f, "\\%03o", *s) < 0)
				return EOF;
		}else if(fputc(*s, f) == EOF){
			return EOF;
		}

	return 0;
}

{'fraction_non_alphanumeric': 0.2241845664280032, 'fraction_numerical': 0.01543357199681782, 'mean_word_length': 2.032320308731307, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4959150', 'n_tokens_mistral': 3177, 'n_tokens_neox': 2701, 'n_words': 628}