<commit_before>#include <stdio.h>

#include "lexer.h"

extern const char * token_value(int tokeType);
extern long scan(const char *source, struct CToken **tokens);

int main(int argc, char *argv[]) {
    printf("test\n");
    struct CToken *tokens = NULL;
    long result = scan("1+2;", &tokens);
    printf("result = %ld\n", result);
    for (int i=0; i<result; ++i) {
        printf("Token {start=%ld, end=%ld, kind=%ld, line=%ld, pos=%ld}\n", tokens[i].start, tokens[i].end, tokens[i].kind, tokens[i].line, tokens[i].pos);
    }
    printf("buh-bye\n");
    return 0;
}<commit_msg>added ability to run lexer on examples/ code<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <cassert>

#include "lexer.h"

extern const char * token_value(int tokeType);
extern long scan(const char *source, struct CToken **tokens);

unsigned int load(const char *filename, char **content) {
    unsigned int size = 0;
    FILE *fd = fopen(filename, "rb");
    if (NULL == fd) {
        *content = NULL;
        return -1;
    }
    fseek(fd, 0, SEEK_END);
    size = ftell(fd);
    fseek(fd, 0, SEEK_SET);
    *content = (char *)malloc(size + 1);
    if (size != fread(*content, sizeof(char), size, fd)) {
        free(*content);
        return -2;
    }
    fclose(fd);
    (*content)[size] = 0;
    return size;
}

void substring(char *str, int start, int end) {
    printf("%.*s", end-start+1, (str+start));
}

int main(int argc, char *argv[]) {
    printf("test\n");
    char *content;
    unsigned int size;
    size = load("../../examples/hello-world.sota", &content);
    assert(size);
    struct CToken *tokens = NULL;
    long count = scan((const char *)&content, &tokens);
    printf("count = %ld\n", count);
    for (int i=0; i<count; ++i) {
        struct CToken token = tokens[i];
        printf("Token {start=%ld end=%ld kind=%ld line=%ld pos=%ld skip=%ld}\n"\
            , token.start\
            , token.end\
            , token.kind\
            , token.line\
            , token.pos\
            , token.skip);
        substring(content, token.start, token.end);
        printf("\n");
    }
    printf("buh-bye\n");
    return 0;
}
{'fraction_non_alphanumeric': 0.1791530944625407, 'fraction_numerical': 0.006514657980456026, 'mean_word_length': 2.7521815008726005, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3780377', 'n_tokens_mistral': 755, 'n_tokens_neox': 735, 'n_words': 196}