<commit_before>/* parser.c: Parser for Tyger files.
 *
 * Copyright: (c) 2016 <NAME> (<EMAIL>)
 * Created:   2016-08-25
 * Version:   $Id: parser.c 127 2017-05-14 17:20:46Z jacco $
 *
 * This software is distributed under the terms of the MIT license. See
 * http://www.opensource.org/licenses/mit-license.php for details.
 */

#include <stdio.h>
#include <string.h>
#include <wchar.h>

#include <libjvs/list.h>
#include <libjvs/buffer.h>
#include <libjvs/hash.h>
#include <libjvs/defs.h>

#include "tokenizer.h"
#include "parser.h"

typedef enum {
    ST_INITIAL,
    ST_NAME,
    ST_EQUALS,
    ST_CONST,
    ST_ARRAY,
    ST_STRUCT,
    ST_ENUM,
    ST_UNION,
} State;

static void expected(tkType type, tkToken *token, Buffer *error)
{
    bufSetF(error, "%s:%d:%d: expected %s, got %s.\n",
            token->file, token->line, token->column,
            tokentype_enum_to_string(type),
            tokentype_enum_to_string(token->type));
}

static Definition *find_def(List *defs, const char *name)
{
    Definition *def;

    for (def = listHead(defs); def; def = listNext(def)) {
        if (strcmp(def->name, name) == 0) break;
    }

    return def;
}

static int expect_token(tkToken **token, tkType type, Buffer *error)
{
    if ((*token)->type != type) {
        expected(type, *token, error);
        return 1;
    }
    else {
        *token = listNext(*token);
        return 0;
    }
}

static int expect_long(tkToken **token, long *contents, Buffer *error)
{
    if ((*token)->type != TT_LONG) {
        expected(TT_LONG, *token, error);
        return 1;
    }
    else {
        *contents = (*token)->u.l;
        *token = listNext(*token);
        return 0;
    }
}

static int expect_float(tkToken **token, double *contents, Buffer *error)
{
    if ((*token)->type != TT_DOUBLE) {
        expected(TT_DOUBLE, *token, error);
        return 1;
    }
    else {
        *contents = (*token)->u.d;
        *token = listNext(*token);
        return 0;
    }
}

static int expect_string(tkToken **token, tkType type, char **contents, Buffer *error)
{
    if ((*token)->type != type) {
        expected(type, *token, error);
        return 1;
    }
    else {
        *contents = (*token)->u.s;
        *token = listNext(*token);
        return 0;
    }
}

static int process_const(Definition *def, List *defs, tkToken **token, Buffer *error)
{
    char *item_type;

    Definition *type_def;

    if (expect_string(token, TT_USTRING, &item_type, error) != 0) {
        fprintf(stderr, "Got const type \"%s\"\n", item_type);
        return 1;
    }
    else if ((type_def = find_def(defs, item_type)) == NULL) {
        bufSetF(error, "%s:%d:%d: unknown type: \"%s\".\n",
                (*token)->file, (*token)->line, (*token)->column, item_type);
        return 1;
    }
    else if (type_def->type == DT_INT) {
        if (expect_long(token, &def->u.const_def.value.l, error) != 0) return 1;
    }
    else if (type_def->type == DT_FLOAT) {
        if (expect_float(token, &def->u.const_def.value.d, error) != 0) return 1;
    }
    else if (type_def->type == DT_ASTRING || type_def->type == DT_USTRING) {
        if (expect_string(token, TT_DSTRING, &def->u.const_def.value.s, error) != 0) return 1;
    }
    else {
        bufSetF(error, "%s:%d:%d: invalid const type %s.\n",
            (*token)->file, (*token)->line, (*token)->column,
            tokentype_enum_to_string(type_def->type));
        return 1;
    }

    def->type = DT_CONST;

    def->u.const_def.const_type = type_def;

    listAppendTail(defs, def);

    return 0;
}

static int process_array(Definition *def, List *defs, tkToken **token, Buffer *error)
{
    char *item_type, *item_name;

    Definition *item_def;

    if (expect_token(token, TT_OPAREN, error) != 0) {
        return 1;<commit_msg>Fixed bug in parser.<commit_after>/* parser.c: Parser for Tyger files.
 *
 * Copyright: (c) 2016 <NAME> (<EMAIL>)
 * Created:   2016-08-25
 * Version:   $Id: parser.c 127 2017-05-14 17:20:46Z jacco $
 *
 * This software is distributed under the terms of the MIT license. See
 * http://www.opensource.org/licenses/mit-license.php for details.
 */

#include <stdio.h>
#include <string.h>
#include <wchar.h>

#include <libjvs/list.h>
#include <libjvs/buffer.h>
#include <libjvs/hash.h>
#include <libjvs/defs.h>

#include "tokenizer.h"
#include "parser.h"

typedef enum {
    ST_INITIAL,
    ST_NAME,
    ST_EQUALS,
    ST_CONST,
    ST_ARRAY,
    ST_STRUCT,
    ST_ENUM,
    ST_UNION,
} State;

static void expected(tkType type, tkToken *token, Buffer *error)
{
    bufSetF(error, "%s:%d:%d: expected %s, got %s.\n",
            token->file, token->line, token->column,
            tokentype_enum_to_string(type),
            tokentype_enum_to_string(token->type));
}

static Definition *find_def(List *defs, const char *name)
{
    Definition *def;

    for (def = listHead(defs); def; def = listNext(def)) {
        if (strcmp(def->name, name) == 0) break;
    }

    return def;
}

static int expect_token(tkToken **token, tkType type, Buffer *error)
{
    if ((*token)->type != type) {
        expected(type, *token, error);
        return 1;
    }
    else {
        *token = listNext(*token);
        return 0;
    }
}

static int expect_long(tkToken **token, long *contents, Buffer *error)
{
    if ((*token)->type != TT_LONG) {
        expected(TT_LONG, *token, error);
        return 1;
    }
    else {
        *contents = (*token)->u.l;
        *token = listNext(*token);
        return 0;
    }
}

static int expect_float(tkToken **token, double *contents, Buffer *error)
{
    if ((*token)->type != TT_DOUBLE) {
        expected(TT_DOUBLE, *token, error);
        return 1;
    }
    else {
        *contents = (*token)->u.d;
        *token = listNext(*token);
        return 0;
    }
}

static int expect_string(tkToken **token, tkType type, char **contents, Buffer *error)
{
    if ((*token)->type != type) {
        expected(type, *token, error);
        return 1;
    }
    else {
        *contents = (*token)->u.s;
        *token = listNext(*token);
        return 0;
    }
}

static int process_const(Definition *def, List *defs, tkToken **token, Buffer *error)
{
    char *item_type;

    Definition *type_def;

    if (expect_string(token, TT_USTRING, &item_type, error) != 0) {
        fprintf(stderr, "Got const type \"%s\"\n", item_type);
        return 1;
    }
    else if ((type_def = find_def(defs, item_type)) == NULL) {
        bufSetF(error, "%s:%d:%d: unknown type: \"%s\".\n",
                (*token)->file, (*token)->line, (*token)->column, item_type);
        return 1;
    }
    else if (type_def->type == DT_INT) {
        if (expect_long(token, &def->u.const_def.value.l, error) != 0) return 1;
    }
    else if (type_def->type == DT_FLOAT) {
        if (expect_float(token, &def->u.const_def.value.d, error) != 0) return 1;
    }
    else if (type_def->type == DT_ASTRING || type_def->type == DT_USTRING) {
        if (expect_string(token, TT_DSTRING, &def->u.const_def.value.s, error) != 0) return 1;
    }
    else {
        bufSetF(error, "%s:%d:%d: invalid const type %s.\n",
            (*token)->file, (*token)->line, (*token)->column,
            deftype_enum_to_string(type_def->type));
        return 1;
    }

    def->type = DT_CONST;

    def->u.const_def.const_type = type_def;

    listAppendTail(defs, def);

    return 0;
}

static int process_array(Definition *def, List *defs, tkToken **token, Buffer *error)
{
    char *item_type, *item_name;

    Definition *item_def;

    if (expect_token(token, TT_OPAREN, error) != 0) {
        return 1;
{'fraction_non_alphanumeric': 0.15922560252864482, 'fraction_numerical': 0.013433425523508494, 'mean_word_length': 2.6422062350119906, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5667104', 'n_tokens_mistral': 2810, 'n_tokens_neox': 2691, 'n_words': 644}