<commit_before>#include <stdio.h> // fprintf, stderr
#include <stdlib.h> // strtod

#include "value.h"
#include "read.h"

inline static bool is_space(char c) {
    return c == ' ' || c == '\r' || c == '\n' || c == '\t';
}

inline static bool is_digit(char c) {
    return c >= '0' && c <= '9';
}

inline static bool is_letter(char c) {
    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}

// R5RS without following: . @
// http://www.schemers.org/Documents/Standards/R5RS/HTML/r5rs-Z-H-5.html#%_sec_2.1
inline static bool is_symbol(char c) {
    return is_digit(c) || is_letter(c) || 
        (c == '!') || (c == '$') || (c == '%') || (c == '&') || 
        (c == '*') || (c == '+') || (c == '-') || (c == ':') || 
        (c == '<') || (c == '=') || (c == '>') || (c == '?') || 
        (c == '^') || (c == '_') || (c == '~') || (c == '/');
}

/* *** */

static char next_char(reader_t* reader) {
    char c = *reader->cur;
    //fprintf(stdout, "Note: %c -> %c\n", *reader->cur, *(reader->cur + 1));
    reader->cur++;
    if (c == '\n') {
        reader->line++;
    }
    return c;
}

static char peek_next_char(reader_t* reader) {
    if (*reader->cur == '\0') return '\0';
    return *(reader->cur + 1);
}

static void eat_whitespace(reader_t* reader) {
    while (is_space(*reader->cur) && (*reader->cur) != '\0') {
        next_char(reader);
    }
    if ((*reader->cur) == ';') {
        while (*reader->cur != '\n' && *reader->cur != '\0') {
            next_char(reader);
        }
    }
    
    if ((*reader->cur) == '\0') {
        reader->toktype = TOK_EOF;
    }
}

static void next_token(reader_t *reader) {
    eat_whitespace(reader);
    
    if ((*reader->cur) == '(') {
        reader->toktype = TOK_LPAREN;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == ')') {
        reader->toktype = TOK_RPAREN;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == '#') {
        reader->toktype = TOK_HASH;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == '.') {
        reader->toktype = TOK_DOT;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == '\'') {
        reader->toktype = TOK_QUOTE;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == '\"') {
        reader->toktype = TOK_STRING;
        reader->tokstart = reader->cur;
    } else if (is_digit(*reader->cur)) {
        reader->toktype = TOK_NUMBER;  
        reader->tokstart = reader->cur;  
    } else if ((*reader->cur) == '\0') {
        reader->toktype = TOK_EOF;
        reader->tokstart = reader->cur;
    } else if (is_symbol(*reader->cur)) {
        reader->toktype = TOK_SYMBOL;
        reader->tokstart = reader->cur;
    } else {
        fprintf(stderr, "Error: Unknown token at line %d (starts with '%c')", reader->line, *reader->cur);
    }
}

static void read1(reader_t *reader);
static void read_list(reader_t *reader);

static void read_number(reader_t *reader){
    double d = strtod(reader->tokstart, NULL);
    
    while (is_digit(*reader->cur)) {
        next_char(reader);
    }

    if (*reader->cur == '.' && is_digit(peek_next_char(reader))) {
        next_char(reader);<commit_msg>Added negative numbers!<commit_after>#include <stdio.h> // fprintf, stderr
#include <stdlib.h> // strtod

#include "value.h"
#include "read.h"

inline static bool is_space(char c) {
    return c == ' ' || c == '\r' || c == '\n' || c == '\t';
}

inline static bool is_digit(char c) {
    return c >= '0' && c <= '9';
}

inline static bool is_letter(char c) {
    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}

// R5RS without following: . @
// http://www.schemers.org/Documents/Standards/R5RS/HTML/r5rs-Z-H-5.html#%_sec_2.1
inline static bool is_symbol(char c) {
    return is_digit(c) || is_letter(c) || 
        (c == '!') || (c == '$') || (c == '%') || (c == '&') || 
        (c == '*') || (c == '+') || (c == '-') || (c == ':') || 
        (c == '<') || (c == '=') || (c == '>') || (c == '?') || 
        (c == '^') || (c == '_') || (c == '~') || (c == '/');
}

/* *** */

static char next_char(reader_t* reader) {
    char c = *reader->cur;
    //fprintf(stdout, "Note: %c -> %c\n", *reader->cur, *(reader->cur + 1));
    reader->cur++;
    if (c == '\n') {
        reader->line++;
    }
    return c;
}

static char peek_next_char(reader_t* reader) {
    if (*reader->cur == '\0') return '\0';
    return *(reader->cur + 1);
}

static void eat_whitespace(reader_t* reader) {
    while (is_space(*reader->cur) && (*reader->cur) != '\0') {
        next_char(reader);
    }
    if ((*reader->cur) == ';') {
        while (*reader->cur != '\n' && *reader->cur != '\0') {
            next_char(reader);
        }
    }
    
    if ((*reader->cur) == '\0') {
        reader->toktype = TOK_EOF;
    }
}

static void next_token(reader_t *reader) {
    eat_whitespace(reader);
    
    if ((*reader->cur) == '(') {
        reader->toktype = TOK_LPAREN;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == ')') {
        reader->toktype = TOK_RPAREN;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == '#') {
        reader->toktype = TOK_HASH;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == '.') {
        reader->toktype = TOK_DOT;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == '\'') {
        reader->toktype = TOK_QUOTE;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == '\"') {
        reader->toktype = TOK_STRING;
        reader->tokstart = reader->cur;
    } else if (is_digit(*reader->cur)) {
        reader->toktype = TOK_NUMBER;  
        reader->tokstart = reader->cur;  
    } else if (((*reader->cur) == '-') && is_digit(peek_next_char(reader))) { 
        reader->toktype = TOK_NUMBER;
        reader->tokstart = reader->cur;
    } else if ((*reader->cur) == '\0') {
        reader->toktype = TOK_EOF;
        reader->tokstart = reader->cur;
    } else if (is_symbol(*reader->cur)) {
        reader->toktype = TOK_SYMBOL;
        reader->tokstart = reader->cur;
    } else {
        fprintf(stderr, "Error: Unknown token at line %d (starts with '%c')", reader->line, *reader->cur);
    }
}

static void read1(reader_t *reader);
static void read_list(reader_t *reader);

static void read_number(reader_t *reader){
    double d = strtod(reader->tokstart, NULL);
    
    while (is_digit(*reader->cur)) {
        next_char(reader);
    }

    if (*reader->cur == '.' && is_digit(peek_next_char(reader))) {
        next_char(reader);
{'fraction_non_alphanumeric': 0.23041825095057034, 'fraction_numerical': 0.005171102661596958, 'mean_word_length': 2.421436004162331, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 131, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3453082', 'n_tokens_mistral': 2418, 'n_tokens_neox': 2381, 'n_words': 479}