<commit_before>//
// Created by Lucas on 10/15/2016.
//

#include <ctype.h>
#include <string.h>
#include <malloc.h>
#include <stdbool.h>
#include "first.h"
#include "readgramar.h"

FirstSet *first(GramaticalRule *headgramaticalrule)
{
    FirstSet *head = NULL;
    int mudou = false;

    //CARACTER_VAZIO esta definido em readgramar.h

    do
    {
        GramaticalRule *currentrule = headgramaticalrule;
        while (currentrule != NULL)
        {
            char key = currentrule->key;
            FirstSet *currentfirstset = seekkey(head,key);
            if (currentfirstset == NULL)
            {
                currentfirstset = (FirstSet*) malloc(sizeof(FirstSet));
                currentfirstset->key = key;
                currentfirstset->set = (char*) malloc(500);
                if (head == NULL)
                    head = currentfirstset;
            }

            for (int i = 0; i < strlen(currentrule->rule); i++)
            {
                FirstSet *notterminalset;
                char crule = currentrule->rule[i];
                if (islower(crule))
                {
                    int length = strlen(currentfirstset->set);
                    currentfirstset->set[ length + 1] = crule;
                    currentfirstset->set[length + 2] = '\0';
                    mudou = true;
                } else if (notterminalset = seekkey(head,crule) != NULL) {
                    char *newfirst = (char*) malloc(500);

<commit_msg>Improving<commit_after>//
// Created by Lucas on 10/15/2016.
//

#include <ctype.h>
#include <string.h>
#include <malloc.h>
#include <stdbool.h>
#include "first.h"
#include "readgramar.h"

FirstSet *first(GramaticalRule *headgramaticalrule)
{
    FirstSet *head = NULL;
    int mudou = false;

    //CARACTER_VAZIO esta definido em readgramar.h

    do
    {
        GramaticalRule *currentrule = headgramaticalrule;
        while (currentrule != NULL)
        {
            char key = currentrule->key;
            FirstSet *currentfirstset = seekkey(head,key);
            if (currentfirstset == NULL)
            {
                currentfirstset = (FirstSet*) malloc(sizeof(FirstSet));
                currentfirstset->key = key;
                currentfirstset->set = (char*) malloc(500);
                if (head == NULL)
                    head = currentfirstset;
            }

            if (strlen(currentrule->rule) >= 1)
            {
                FirstSet *notterminalset;
                char crule = currentrule->rule[1];
                if (islower(crule))
                {
                    int length = strlen(currentfirstset->set);
                    currentfirstset->set[ length + 1] = crule;
                    currentfirstset->set[length + 2] = '\0';
                    mudou = true;
                } else if (notterminalset = seekkey(head,crule) != NULL) {
                    char *newfirst = (char *) malloc(500);
                    copyrulewithoutempty(notterminalset, newfirst);




{'fraction_non_alphanumeric': 0.11426662211827597, 'fraction_numerical': 0.012362178416304711, 'mean_word_length': 1.642541924095322, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4562242', 'n_tokens_mistral': 941, 'n_tokens_neox': 846, 'n_words': 190}