<commit_before>/*
 * cx_generator.c
 *
 *  Created on: Sep 17, 2012
 *      Author: sander
 */

#include "corto.h"
#include "cx_generator.h"
#include "ctype.h"
#include "stdarg.h"
#include "stdio.h"

#ifdef CX_GENERATOR

/* Generator functions */
cx_generator gen_new(cx_string name, cx_string language) {
    cx_generator result;

    result = cx_alloc(sizeof(struct cx_generator_s));

    /* List of output directories is initially empty */
    result->attributes = NULL;

    /* List of files is initially empty */
    result->files = NULL;

    /* No imports are resolved */
    result->imports = NULL;

    /* Set objects */
    result->objects = NULL;

    /* Set name */
    if (name) {
        result->name = cx_strdup(name);
    } else {
        result->name = NULL;
    }

    if (language) {
        result->language = cx_strdup(language);
    } else {
        result->language = cx_strdup("c"); /* Take 'c' as default language */
    }

    /* Set id-generation to default */
    result->idKind = CX_GENERATOR_ID_DEFAULT;

    /* Set action-callbacks */
    result->start_action = NULL;
    result->id_action = NULL;

    /* No library loaded */
    result->library = NULL;

    /* Current will be set by object walk */
    result->current = NULL;

    return result;
}

/* Control how id's are generated */
g_idKind g_setIdKind(cx_generator g, g_idKind kind) {
    g_idKind prev;
    prev = g->idKind;
    g->idKind = kind;
    return prev;
}

/* Get name, or if no name is provided, return name of current parse-object */
cx_string g_getName(cx_generator g) {
    cx_string result;

    result = NULL;
    if (g->name) {
        result = g->name;
    } else if (g->current) {
        result = cx_nameof(g->current->o);
    }

    return result;
}

/* Get current object */
cx_object g_getCurrent(cx_generator g) {
    cx_object result;

    result = NULL;

    if (g->current) {
        result = g->current->o;
    }

    return result;
}

/* Add to-parse object */
void gen_parse(cx_generator g, cx_object object, cx_bool parseSelf, cx_bool parseScope, cx_string prefix) {
    g_object* o = NULL;
    cx_iter objectIter;

    /* First do a lookup, check if the object already exists */
    if (g->objects) {
        g_object *object;
        objectIter = cx_llIter(g->objects);
        while(cx_iterHasNext(&objectIter)) {
            object = cx_iterNext(&objectIter);
            if (object->o == object) {
                o = object;
                break;
            }
        }
    }

    if (!o) {
        o = cx_alloc(sizeof(g_object));
        cx_claim(object);
        o->o = object;
        o->parseSelf = parseSelf;
        o->parseScope = parseScope;

        if (prefix) {
            if (strlen(prefix) >= sizeof(cx_id)) {
                cx_error("prefix cannot be longer than %d characters", sizeof(cx_id));
                o->prefix = NULL;
            } else {
                o->prefix = cx_strdup(prefix);
            }
        } else {
            o->prefix = NULL;
        }

        if (!g->objects) {
            g->objects = cx_llNew();
        }
        cx_llAppend(g->objects, o);

        if (!g->current) {
            g->current = o;
        }
    } else {
        /* Prefixes can be overridden if NULL. */
        if (!o->prefix) {
            o->prefix = cx_strdup(prefix);
        }
    }
}

static int g_genAttributeFind(void *value, void *userData) {
    g_attribute *attr = value;
    if(!strcmp(attr->key, *(cx_string*)userData)) {
        *(void**)userData = attr;
        return 0;
    }
    return 1;
}

/* Set attribute */
void gen_setAttribute(cx_generator g, cx_string key, cx_string value) {
    g_attribute* attr = NULL;

    if (!g->attributes) {
        g->attributes = cx_llNew();
    }else {
        void* userData = key;
        if(!cx_llWalk(g->attributes, g_genAttributeFind, &userData)) {
            attr = userData;
        }
    }

    if(!attr) {
        attr = cx_alloc(sizeof(g_attribute));
        attr->key = cx_strdup(key);
    }else {
        cx_dealloc(attr->value);
    }
    attr->value = cx_strdup(value);

    cx_llAppend(g->attributes, attr);
}

/* Get attribute */
cx_string gen_getAttribute(cx_generator g, cx_string key) {
    cx_string result = NULL;

    if(g->attributes) {
        void *userData = key;
        if(!cx_llWalk(g->attributes, g_genAttributeFind, &userData)) {
            result = ((g_attribute*)userData)->value;
        }
    }

    if (!result) {
        result = "";
    }

    return result;
}

/* Load generator actions from library */
cx_int16 gen_load(cx_generator g, cx_string library) {

    /* Load library from generator path */
    cx_string relativePath = cx_envparse("generators/lib%s.so", library);
    cx_string path = cx_locateLib(relativePath);
    if (!path) {
        cx_error("generator '%s' not found", relativePath);
        goto error;
    }

    g->library = cx_dlOpen(path);
    cx_dealloc(path);
    if (!g->library) {
        cx_error("%s", cx_dlError());
        goto error;
    }

    /* Load actions */
    g->start_action = (g_startAction)cx_dlProc(g->library, "corto_genMain");
    if (!g->start_action) {
        cx_error("g_Load: unresolved symbol 'corto_genMain'");
        goto error;
    }
    g->id_action = (g_idAction)cx_dlProc(g->library, "corto_genId");

    /* Function is allowed to be absent. */

    cx_dealloc(relativePath);
    return 0;
error:
    cx_dealloc(relativePath);
    return -1;
}

static int g_freeObjects(void* _o, void* udata) {
    g_object* o;

    CX_UNUSED(udata);

    o = _o;
    if (o->prefix) {
        cx_dealloc(o->prefix);
    }
    cx_release(o->o);
    cx_dealloc(o);

    return 1;
}

/* Free snippet */
static int g_freeSnippet(void* o, void* udata) {
    g_fileSnippet* snippet;
    g_file file;

    snippet = o;
    file = udata;

    if (!snippet->used) {
        g_fileWrite(file, "%s(%s)", snippet->option, snippet->id);
        g_fileWrite(file, "%s", snippet->src);
        g_fileWrite(file, "$end\n");
        cx_warning("%s: code-snippet '%s' is not used, manually merge or remove from file.", file->name, snippet->id);
    }

    if (snippet->id) {
        cx_dealloc(snippet->id);
    }
    if (snippet->src) {
        cx_dealloc(snippet->src);
    }

    cx_dealloc(snippet);

    return 1;
}

/* Close file */
static int g_closeFile(void* o, void* udata) {
    g_file file;
    CX_UNUSED(udata);

    file = o;

    g_fileClose(file);

    return 1;
}

static int g_freeAttribute(void* _o, void* udata) {
    g_attribute* o;

    CX_UNUSED(udata);

    o = _o;
    if (o->key) {
        cx_dealloc(o->key);
    }
    if (o->value) {
        cx_dealloc(o->value);
    }

    cx_dealloc(o);

    return 1;
}

/* Free generator */
void gen_free(cx_generator g) {
    if (g->library) {
        cx_dlClose(g->library);
        g->library = NULL;
    }

    if (g->objects) {
        cx_llWalk(g->objects, g_freeObjects, NULL);
        cx_llFree(g->objects);
        g->objects = NULL;
    }

    if (g->files) {
        cx_llWalk(g->files, g_closeFile, NULL);
        cx_llFree(g->files);
        g->files = NULL;
    }

    if (g->attributes) {
        cx_llWalk(g->attributes, g_freeAttribute, NULL);
        cx_llFree(g->attributes);
        g->attributes = NULL;
    }

    if (g->imports) {
        cx_llFree(g->imports);
    }

    if (g->name) {
        cx_dealloc(g->name);
    }
    if (g->language) {
        cx_dealloc(g->language);
    }
    cx_dealloc(g);
}

/* Start generator */
cx_int16 gen_start(cx_generator g) {
    return g->start_action(g);
}

/* ==== Generator utility functions */
typedef struct g_serializeImports_t {
    cx_generator g;
    cx_object stack[1024]; /* Maximum serializer-depth */
    cx_uint32 count;
}g_serializeImports_t;
cx_int16 g_serializeImportsReference(cx_serializer s, cx_value *v, void* userData) {
    cx_object o;
    g_serializeImports_t *data = userData;
    cx_generator g = data->g;

    o = *(cx_object*)cx_valueValue(v);
    if (o) {
        /* Search unscoped object for references to other modules */
        if (!cx_checkAttr(o, CX_ATTR_SCOPED)) {
            cx_uint32 i;

            /* Make sure to not serialize an object twice, in case of a cycle */
            for(i=0; i<data->count; i++) {
                if (data->stack[i] == o) {
                    break;
                }
            }

            if (i == data->count) {
                data->stack[data->count] = o;
                data->count++;
                cx_serialize(s, o, userData);
                data->count--;
            }
        } else {
            if (!g_mustParse(g, o)) {
                cx_object parent = o;

                while(!cx_instanceof(cx_type(cx_package_o), parent)) {
                    parent = cx_parentof(parent);
                }

                if ((parent != root_o) && (parent != corto_o) && (parent != corto_lang_o)) {
                    if (!g->imports) {
                        g->imports = cx_llNew();
                    }
                    if (!cx_llHasObject(g->imports, parent)) {
                        cx_llInsert(g->imports, parent);
                    }
                }
            }
        }
    }

    return 0;
}

struct cx_serializer_s g_serializeImportsSerializer(void) {
    struct cx_serializer_s result;
    cx_serializerInit(&result);
    result.reference = g_serializeImportsReference;
    result.access = CX_PRIVATE;
    result.accessKind = CX_NOT; /* Serialize not nothing, thus everything. */
    return result;
}

struct g_walkObjects_t {
    g_walkAction action;
    void* userData;
};

/* Resolve imports */
int g_importWalk(cx_object o, void* userData) {
    cx_generator g = userData;
    g_serializeImports_t walkData;
    struct cx_serializer_s s;

    walkData.count = 0;
    walkData.g = g;
    s = g_serializeImportsSerializer();
    cx_serialize(&s, o, &walkData);
    return 1;
}

cx_int16 g_resolveImports(cx_generator generator) {
    g_walkRecursive(generator, g_importWalk, generator);
    return 0;
}


/* Recursively walk scopes */
int g_walkObjects(void* o, void* userData) {
    struct g_walkObjects_t* data;

    data = userData;

    if (!data->action(o, data->userData)) {
        goto stop;
    }

    return cx_scopeWalk(o, g_walkObjects, userData);
stop:
    return 0;
}

/* Walk objects, choose between recursive scopewalk or only top-level objects */
static int g_walk_ext(cx_generator g, g_walkAction action, void* userData, cx_bool scopeWalk, cx_bool recursiveWalk) {
    cx_iter iter;

    if (g->objects) {
        g_object* o;

        iter = cx_llIter(g->objects);
        while(cx_iterHasNext(&iter)) {
            o = cx_iterNext(&iter);

            /* Parse object */
            if (o->parseSelf) {
                g->current = o;
                if (!action(o->o, userData)) {
                    goto stop;
                }
            }
            /* Walk scopes */
            if (o->parseScope && scopeWalk) {
                g->current = o;
                if (!recursiveWalk) {
                    if (!cx_scopeWalk(o->o, action, userData)) {
                        goto stop;
                    }
                } else {
                    struct g_walkObjects_t walkData;
                    walkData.action = action;
                    walkData.userData = userData;

                    /* Recursively walk scopes */
                    if (!cx_scopeWalk(o->o, g_walkObjects, &walkData)) {
                        goto stop;
                    }
                }
            }
        }
    }

    return 1;
stop:
    return 0;
}

/* Walk objects */
int g_walk(cx_generator g, g_walkAction action, void* userData) {
    return g_walk_ext(g, action, userData, TRUE, FALSE);
}

/* Walk objects, never walk scopes, even if object is required to. */
int g_walkNoScope(cx_generator g, g_walkAction action, void* userData) {
    return g_walk_ext(g, action, userData, FALSE, FALSE);
}

/* Walk objects recursively */
int g_walkRecursive(cx_generator g, g_walkAction action, void* userData) {
    return g_walk_ext(g, action, userData, TRUE, TRUE);
}

/* Find prefix for a given object. Search parse-object in generator
 * which is closest to the object passed to this function. */
static g_object* g_findPrefix(cx_generator g, cx_object o, cx_object* match) {
    cx_iter iter;
    g_object *result, *t;
    cx_object parent;
    unsigned int distance, minDistance;

    result = NULL;
    if (g->objects) {
        minDistance = -1;
        iter = cx_llIter(g->objects);
        while(cx_iterHasNext(&iter)) {
            t = cx_iterNext(&iter);

            /* Check if object occurs in scope of 'o' and measure distance to 'o' */
            parent = o;
            distance = 0;
            while(parent && (parent != t->o)) {
                distance++;
                parent = cx_parentof(parent);
            }

            /* If a parent was found (parent of root is NULL), assign it to result if
             * distance is smaller than minDistance */
            if (parent) {
                if ((distance < minDistance)) {
                    result = t;
                    minDistance = distance;
                    if (match) {
                        *match = parent;
                    }
                }
            }
        }
    }

    return result;
}

/* Obtain prefix */
cx_string g_getPrefix(cx_generator g, cx_object o) {
    g_object* prefix;

    /* Lookup prefix */
    prefix = g_findPrefix(g, o, NULL);

    return (prefix != NULL) ? prefix->prefix : NULL;
}

/* Object transformations */
static cx_char* g_oidTransform(cx_generator g, cx_object o, cx_id _id, g_idKind kind) {
    CX_UNUSED(g);

    /* If the object is a function with an argumentlist, cut the argumentlist
     * from the name if the function is not overloaded. This keeps processing for
     * generators trivial. */
    if (cx_class_instanceof(cx_procedure_o, cx_typeof(o))) {
        if (!cx_function(o)->overloaded) {
            cx_char* ptr;
            ptr = strchr(_id, '(');
            if (ptr) {
                *ptr = '\0';
            } else {
                if (cx_procedure(cx_typeof(o))->kind != CX_OBSERVER) {
                    cx_id id;
                    cx_warning("function object '%s' without argument list.", cx_fullname(o, id));
                }
            }
        } else {
            /* If function is overloaded, construct the 'request' string, that is, the string without
             * the argument-names. This results in a string with only the types, which is enough to
             * generate unique names in languages which do not support overloading. */
            cx_id tmp, buff;
            cx_int32 count, i;
            strcpy(tmp, _id);

            cx_signatureName(tmp, _id);
            strcat(_id, "(");

            count = cx_signatureParamCount(tmp);
            if (count == -1) {
                cx_error("invalid signature '%s'", tmp);
                goto error;
            }

            /* strcat is not the most efficient function here, but it is the easiest, and this
             * part of the code is not performance-critical. */
            for(i=0; i<count; i++) {
                cx_signatureParamType(tmp, i, buff, NULL);
                if (i) {
                    strcat(_id, ",");
                }
                strcat(_id, buff);
            }
            strcat(_id, ")");
        }
    }

    /* Check if class-identifiers must be altered */
    if (kind != CX_GENERATOR_ID_DEFAULT) {
        cx_object i = o;
        cx_char* ptr;

        ptr = _id + strlen(_id);
        while(i) {
            while((ptr > _id) && (*ptr != ':')) {
                ptr--;
            }
            if ((cx_class_instanceof(cx_interface_o, i) && cx_type(i)->reference) || (i == cx_type(cx_object_o))) {
                cx_char *start;
                if (*ptr == ':') {
                    start = ptr + 1;
                } else {
                    start = ptr;
                }
                if (kind == CX_GENERATOR_ID_CLASS_UPPER) {
                    *start = toupper(*start);
                } else {
                    *start = tolower(*start);
                }
            }

            if (ptr == _id) {
                break;
            }

            i = cx_parentof(i);
            if (i) {
                if (*ptr == ':') {
                    ptr -= 2;
                }
            }
        }
    }

    return _id;
error:
    return NULL;
}

/* Translate object-id */
cx_string g_fullOidExt(cx_generator g, cx_object o, cx_id id, g_idKind kind) {
    g_object* prefix;
    cx_object match;
    cx_id _id;

    id[0] = '\0';

    /* Find prefix for object */
    match = NULL;
    prefix = g_findPrefix(g, o, &match);

    /* TODO: prefix i.c.m. !CX_GENERATOR_ID_DEFAULT & nested classes i.c.m. !CX_GENERATOR_ID_DEFAULT */

    /* If prefix is found, replace the scope up until the found object with the prefix */
    if (prefix && prefix->prefix) {
        cx_uint32 count;
        cx_object scopes[CX_MAX_SCOPE_DEPTH];

        /* Obtain list of scopes from matched to object */
        count = 0;
        scopes[count] = o;
        while(scopes[count] != match) {
            scopes[count+1] = cx_parentof(scopes[count]);
            count++;
        }

        /* Paste in prefix */
        strcpy(_id, prefix->prefix);
        while(count) {
            count--;
            strcat(_id, "::");
            strcat(_id, cx_nameof(scopes[count]));
        }

    /* If no prefix is found for object, just use the scoped identifier */
    } else {
        cx_fullname(o, _id);
    }

    g_oidTransform(g, o, _id, kind);

    if (g->id_action) {
        g->id_action(_id, id);
    } else {
        strcpy(_id, id);
    }

    return id;
}<commit_msg>#316 Fixed bug in generator<commit_after>/*
 * cx_generator.c
 *
 *  Created on: Sep 17, 2012
 *      Author: sander
 */

#include "corto.h"
#include "cx_generator.h"
#include "ctype.h"
#include "stdarg.h"
#include "stdio.h"

#ifdef CX_GENERATOR

/* Generator functions */
cx_generator gen_new(cx_string name, cx_string language) {
    cx_generator result;

    result = cx_alloc(sizeof(struct cx_generator_s));

    /* List of output directories is initially empty */
    result->attributes = NULL;

    /* List of files is initially empty */
    result->files = NULL;

    /* No imports are resolved */
    result->imports = NULL;

    /* Set objects */
    result->objects = NULL;

    /* Set name */
    if (name) {
        result->name = cx_strdup(name);
    } else {
        result->name = NULL;
    }

    if (language) {
        result->language = cx_strdup(language);
    } else {
        result->language = cx_strdup("c"); /* Take 'c' as default language */
    }

    /* Set id-generation to default */
    result->idKind = CX_GENERATOR_ID_DEFAULT;

    /* Set action-callbacks */
    result->start_action = NULL;
    result->id_action = NULL;

    /* No library loaded */
    result->library = NULL;

    /* Current will be set by object walk */
    result->current = NULL;

    return result;
}

/* Control how id's are generated */
g_idKind g_setIdKind(cx_generator g, g_idKind kind) {
    g_idKind prev;
    prev = g->idKind;
    g->idKind = kind;
    return prev;
}

/* Get name, or if no name is provided, return name of current parse-object */
cx_string g_getName(cx_generator g) {
    cx_string result;

    result = NULL;
    if (g->name) {
        result = g->name;
    } else if (g->current) {
        result = cx_nameof(g->current->o);
    }

    return result;
}

/* Get current object */
cx_object g_getCurrent(cx_generator g) {
    cx_object result;

    result = NULL;

    if (g->current) {
        result = g->current->o;
    }

    return result;
}

/* Add to-parse object */
void gen_parse(cx_generator g, cx_object object, cx_bool parseSelf, cx_bool parseScope, cx_string prefix) {
    g_object* o = NULL;
    cx_iter objectIter;

    /* First do a lookup, check if the object already exists */
    if (g->objects) {
        g_object *object;
        objectIter = cx_llIter(g->objects);
        while(cx_iterHasNext(&objectIter)) {
            object = cx_iterNext(&objectIter);
            if (object->o == object) {
                o = object;
                break;
            }
        }
    }

    if (!o) {
        o = cx_alloc(sizeof(g_object));
        cx_claim(object);
        o->o = object;
        o->parseSelf = parseSelf;
        o->parseScope = parseScope;

        if (prefix) {
            if (strlen(prefix) >= sizeof(cx_id)) {
                cx_error("prefix cannot be longer than %d characters", sizeof(cx_id));
                o->prefix = NULL;
            } else {
                o->prefix = cx_strdup(prefix);
            }
        } else {
            o->prefix = NULL;
        }

        if (!g->objects) {
            g->objects = cx_llNew();
        }
        cx_llAppend(g->objects, o);

        if (!g->current) {
            g->current = o;
        }
    } else {
        /* Prefixes can be overridden if NULL. */
        if (!o->prefix) {
            o->prefix = cx_strdup(prefix);
        }
    }
}

static int g_genAttributeFind(void *value, void *userData) {
    g_attribute *attr = value;
    if(!strcmp(attr->key, *(cx_string*)userData)) {
        *(void**)userData = attr;
        return 0;
    }
    return 1;
}

/* Set attribute */
void gen_setAttribute(cx_generator g, cx_string key, cx_string value) {
    g_attribute* attr = NULL;

    if (!g->attributes) {
        g->attributes = cx_llNew();
    }else {
        void* userData = key;
        if(!cx_llWalk(g->attributes, g_genAttributeFind, &userData)) {
            attr = userData;
        }
    }

    if(!attr) {
        attr = cx_alloc(sizeof(g_attribute));
        attr->key = cx_strdup(key);
    }else {
        cx_dealloc(attr->value);
    }
    attr->value = cx_strdup(value);

    cx_llAppend(g->attributes, attr);
}

/* Get attribute */
cx_string gen_getAttribute(cx_generator g, cx_string key) {
    cx_string result = NULL;

    if(g->attributes) {
        void *userData = key;
        if(!cx_llWalk(g->attributes, g_genAttributeFind, &userData)) {
            result = ((g_attribute*)userData)->value;
        }
    }

    if (!result) {
        result = "";
    }

    return result;
}

/* Load generator actions from library */
cx_int16 gen_load(cx_generator g, cx_string library) {

    /* Load library from generator path */
    cx_string relativePath = cx_envparse("generators/lib%s.so", library);
    cx_string path = cx_locateLib(relativePath);
    if (!path) {
        cx_error("generator '%s' not found", relativePath);
        goto error;
    }

    g->library = cx_dlOpen(path);
    cx_dealloc(path);
    if (!g->library) {
        cx_error("%s", cx_dlError());
        goto error;
    }

    /* Load actions */
    g->start_action = (g_startAction)cx_dlProc(g->library, "corto_genMain");
    if (!g->start_action) {
        cx_error("g_Load: unresolved symbol 'corto_genMain'");
        goto error;
    }
    g->id_action = (g_idAction)cx_dlProc(g->library, "corto_genId");

    /* Function is allowed to be absent. */

    cx_dealloc(relativePath);
    return 0;
error:
    cx_dealloc(relativePath);
    return -1;
}

static int g_freeObjects(void* _o, void* udata) {
    g_object* o;

    CX_UNUSED(udata);

    o = _o;
    if (o->prefix) {
        cx_dealloc(o->prefix);
    }
    cx_release(o->o);
    cx_dealloc(o);

    return 1;
}

/* Free snippet */
static int g_freeSnippet(void* o, void* udata) {
    g_fileSnippet* snippet;
    g_file file;

    snippet = o;
    file = udata;

    if (!snippet->used) {
        g_fileWrite(file, "%s(%s)", snippet->option, snippet->id);
        g_fileWrite(file, "%s", snippet->src);
        g_fileWrite(file, "$end\n");
        cx_warning("%s: code-snippet '%s' is not used, manually merge or remove from file.", file->name, snippet->id);
    }

    if (snippet->id) {
        cx_dealloc(snippet->id);
    }
    if (snippet->src) {
        cx_dealloc(snippet->src);
    }

    cx_dealloc(snippet);

    return 1;
}

/* Close file */
static int g_closeFile(void* o, void* udata) {
    g_file file;
    CX_UNUSED(udata);

    file = o;

    g_fileClose(file);

    return 1;
}

static int g_freeAttribute(void* _o, void* udata) {
    g_attribute* o;

    CX_UNUSED(udata);

    o = _o;
    if (o->key) {
        cx_dealloc(o->key);
    }
    if (o->value) {
        cx_dealloc(o->value);
    }

    cx_dealloc(o);

    return 1;
}

/* Free generator */
void gen_free(cx_generator g) {
    if (g->library) {
        cx_dlClose(g->library);
        g->library = NULL;
    }

    if (g->objects) {
        cx_llWalk(g->objects, g_freeObjects, NULL);
        cx_llFree(g->objects);
        g->objects = NULL;
    }

    if (g->files) {
        cx_llWalk(g->files, g_closeFile, NULL);
        cx_llFree(g->files);
        g->files = NULL;
    }

    if (g->attributes) {
        cx_llWalk(g->attributes, g_freeAttribute, NULL);
        cx_llFree(g->attributes);
        g->attributes = NULL;
    }

    if (g->imports) {
        cx_llFree(g->imports);
    }

    if (g->name) {
        cx_dealloc(g->name);
    }
    if (g->language) {
        cx_dealloc(g->language);
    }
    cx_dealloc(g);
}

/* Start generator */
cx_int16 gen_start(cx_generator g) {
    return g->start_action(g);
}

/* ==== Generator utility functions */
typedef struct g_serializeImports_t {
    cx_generator g;
    cx_object stack[1024]; /* Maximum serializer-depth */
    cx_uint32 count;
}g_serializeImports_t;
cx_int16 g_serializeImportsReference(cx_serializer s, cx_value *v, void* userData) {
    cx_object o;
    g_serializeImports_t *data = userData;
    cx_generator g = data->g;

    o = *(cx_object*)cx_valueValue(v);
    if (o) {
        /* Search unscoped object for references to other modules */
        if (!cx_checkAttr(o, CX_ATTR_SCOPED)) {
            cx_uint32 i;

            /* Make sure to not serialize an object twice, in case of a cycle */
            for(i=0; i<data->count; i++) {
                if (data->stack[i] == o) {
                    break;
                }
            }

            if (i == data->count) {
                data->stack[data->count] = o;
                data->count++;
                cx_serialize(s, o, userData);
                data->count--;
            }
        } else {
            if (!g_mustParse(g, o)) {
                cx_object parent = o;

                while(!cx_instanceof(cx_type(cx_package_o), parent)) {
                    parent = cx_parentof(parent);
                }

                if ((parent != root_o) && (parent != corto_o) && (parent != corto_lang_o)) {
                    if (!g->imports) {
                        g->imports = cx_llNew();
                    }
                    if (!cx_llHasObject(g->imports, parent)) {
                        cx_llInsert(g->imports, parent);
                    }
                }
            }
        }
    }

    return 0;
}

struct cx_serializer_s g_serializeImportsSerializer(void) {
    struct cx_serializer_s result;
    cx_serializerInit(&result);
    result.reference = g_serializeImportsReference;
    result.access = CX_PRIVATE;
    result.accessKind = CX_NOT; /* Serialize not nothing, thus everything. */
    return result;
}

struct g_walkObjects_t {
    g_walkAction action;
    void* userData;
};

/* Resolve imports */
int g_importWalk(cx_object o, void* userData) {
    cx_generator g = userData;
    g_serializeImports_t walkData;
    struct cx_serializer_s s;

    walkData.count = 0;
    walkData.g = g;
    s = g_serializeImportsSerializer();
    cx_serialize(&s, o, &walkData);
    return 1;
}

cx_int16 g_resolveImports(cx_generator generator) {
    g_walkRecursive(generator, g_importWalk, generator);
    return 0;
}


/* Recursively walk scopes */
int g_walkObjects(void* o, void* userData) {
    struct g_walkObjects_t* data;

    data = userData;

    if (!data->action(o, data->userData)) {
        goto stop;
    }

    return cx_scopeWalk(o, g_walkObjects, userData);
stop:
    return 0;
}

/* Walk objects, choose between recursive scopewalk or only top-level objects */
static int g_walk_ext(cx_generator g, g_walkAction action, void* userData, cx_bool scopeWalk, cx_bool recursiveWalk) {
    cx_iter iter;

    if (g->objects) {
        g_object* o;

        iter = cx_llIter(g->objects);
        while(cx_iterHasNext(&iter)) {
            o = cx_iterNext(&iter);

            /* Parse object */
            if (o->parseSelf) {
                g->current = o;
                if (!action(o->o, userData)) {
                    goto stop;
                }
            }
            /* Walk scopes */
            if (o->parseScope && scopeWalk) {
                g->current = o;
                if (!recursiveWalk) {
                    if (!cx_scopeWalk(o->o, action, userData)) {
                        goto stop;
                    }
                } else {
                    struct g_walkObjects_t walkData;
                    walkData.action = action;
                    walkData.userData = userData;

                    /* Recursively walk scopes */
                    if (!cx_scopeWalk(o->o, g_walkObjects, &walkData)) {
                        goto stop;
                    }
                }
            }
        }
    }

    return 1;
stop:
    return 0;
}

/* Walk objects */
int g_walk(cx_generator g, g_walkAction action, void* userData) {
    return g_walk_ext(g, action, userData, TRUE, FALSE);
}

/* Walk objects, never walk scopes, even if object is required to. */
int g_walkNoScope(cx_generator g, g_walkAction action, void* userData) {
    return g_walk_ext(g, action, userData, FALSE, FALSE);
}

/* Walk objects recursively */
int g_walkRecursive(cx_generator g, g_walkAction action, void* userData) {
    return g_walk_ext(g, action, userData, TRUE, TRUE);
}

/* Find prefix for a given object. Search parse-object in generator
 * which is closest to the object passed to this function. */
static g_object* g_findPrefix(cx_generator g, cx_object o, cx_object* match) {
    cx_iter iter;
    g_object *result, *t;
    cx_object parent;
    unsigned int distance, minDistance;

    result = NULL;
    if (g->objects) {
        minDistance = -1;
        iter = cx_llIter(g->objects);
        while(cx_iterHasNext(&iter)) {
            t = cx_iterNext(&iter);

            /* Check if object occurs in scope of 'o' and measure distance to 'o' */
            parent = o;
            distance = 0;
            while(parent && (parent != t->o)) {
                distance++;
                parent = cx_parentof(parent);
            }

            /* If a parent was found (parent of root is NULL), assign it to result if
             * distance is smaller than minDistance */
            if (parent) {
                if ((distance < minDistance)) {
                    result = t;
                    minDistance = distance;
                    if (match) {
                        *match = parent;
                    }
                }
            }
        }
    }

    return result;
}

/* Obtain prefix */
cx_string g_getPrefix(cx_generator g, cx_object o) {
    g_object* prefix;

    /* Lookup prefix */
    prefix = g_findPrefix(g, o, NULL);

    return (prefix != NULL) ? prefix->prefix : NULL;
}

/* Object transformations */
static cx_char* g_oidTransform(cx_generator g, cx_object o, cx_id _id, g_idKind kind) {
    CX_UNUSED(g);

    /* If the object is a function with an argumentlist, cut the argumentlist
     * from the name if the function is not overloaded. This keeps processing for
     * generators trivial. */
    if (cx_class_instanceof(cx_procedure_o, cx_typeof(o))) {
        if (!cx_function(o)->overloaded) {
            cx_char* ptr;
            ptr = strchr(_id, '(');
            if (ptr) {
                *ptr = '\0';
            } else {
                if (cx_procedure(cx_typeof(o))->kind != CX_OBSERVER) {
                    cx_id id;
                    cx_warning("function object '%s' without argument list.", cx_fullname(o, id));
                }
            }
        } else {
            /* If function is overloaded, construct the 'request' string, that is, the string without
             * the argument-names. This results in a string with only the types, which is enough to
             * generate unique names in languages which do not support overloading. */
            cx_id tmp, buff;
            cx_int32 count, i;
            strcpy(tmp, _id);

            cx_signatureName(tmp, _id);
            strcat(_id, "(");

            count = cx_signatureParamCount(tmp);
            if (count == -1) {
                cx_error("invalid signature '%s'", tmp);
                goto error;
            }

            /* strcat is not the most efficient function here, but it is the easiest, and this
             * part of the code is not performance-critical. */
            for(i=0; i<count; i++) {
                cx_signatureParamType(tmp, i, buff, NULL);
                if (i) {
                    strcat(_id, ",");
                }
                strcat(_id, buff);
            }
            strcat(_id, ")");
        }
    }

    /* Check if class-identifiers must be altered */
    if (kind != CX_GENERATOR_ID_DEFAULT) {
        cx_object i = o;
        cx_char* ptr;

        ptr = _id + strlen(_id);
        while(i) {
            while((ptr > _id) && (*ptr != ':')) {
                ptr--;
            }
            if ((cx_class_instanceof(cx_interface_o, i) && cx_type(i)->reference) || (i == cx_type(cx_object_o))) {
                cx_char *start;
                if (*ptr == ':') {
                    start = ptr + 1;
                } else {
                    start = ptr;
                }
                if (kind == CX_GENERATOR_ID_CLASS_UPPER) {
                    *start = toupper(*start);
                } else {
                    *start = tolower(*start);
                }
            }

            if (ptr == _id) {
                break;
            }

            i = cx_parentof(i);
            if (i) {
                if (*ptr == ':') {
                    ptr -= 2;
                }
            }
        }
    }

    return _id;
error:
    return NULL;
}

/* Translate object-id */
cx_string g_fullOidExt(cx_generator g, cx_object o, cx_id id, g_idKind kind) {
    g_object* prefix;
    cx_object match;
    cx_id _id;

    id[0] = '\0';

    /* Find prefix for object */
    match = NULL;
    prefix = g_findPrefix(g, o, &match);

    /* TODO: prefix i.c.m. !CX_GENERATOR_ID_DEFAULT & nested classes i.c.m. !CX_GENERATOR_ID_DEFAULT */

    /* If prefix is found, replace the scope up until the found object with the prefix */
    if (prefix && prefix->prefix) {
        cx_uint32 count;
        cx_object scopes[CX_MAX_SCOPE_DEPTH];

        /* Obtain list of scopes from matched to object */
        count = 0;
        scopes[count] = o;
        while(scopes[count] != match) {
            scopes[count+1] = cx_parentof(scopes[count]);
            count++;
        }

        /* Paste in prefix */
        strcpy(_id, prefix->prefix);
        while(count) {
            count--;
            strcat(_id, "::");
            strcat(_id, cx_nameof(scopes[count]));
        }

    /* If no prefix is found for object, just use the scoped identifier */
    } else {
        cx_fullname(o, _id);
    }

    g_oidTransform(g, o, _id, kind);

    if (g->id_action) {
        g->id_action(_id, id);
    } else {
        strcpy(id, _id);
    }

    return id;
}
{'fraction_non_alphanumeric': 0.12313956753720864, 'fraction_numerical': 0.0030609379387812412, 'mean_word_length': 1.8064465284892426, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 267, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2063060', 'n_tokens_mistral': 11524, 'n_tokens_neox': 11041, 'n_words': 2740}