<commit_before>#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <glob.h>
#include "libinjection.h"

char g_test[4096];
char g_input[4096];
char g_expected[4096];

size_t modp_rtrim(char* str, size_t len)
{
    while (len) {
        char c = str[len -1];
        if (c == ' ' || c == '\n' || c == '\t' || c == '\r') {
            str[len -1] = '\0';
            len -= 1;
        } else {
            break;
        }
    }
    return len;
}

size_t print_string(char* buf, size_t len, stoken_t* t)
{
    /* print opening quote */
    if (t->str_open != '\0') {
        len += sprintf(buf + len, "%c", t->str_open);
    }

    /* print content */
    len += sprintf(buf + len, "%s", t->val);

    /* print closing quote */
    if (t->str_close != '\0') {
        len += sprintf(buf + len, "%c", t->str_close);
    }

    return len;
}

size_t print_var(char* buf, size_t len, stoken_t* t)
{
    if (t->count >= 1) {
        len += sprintf(buf + len, "%c", '@');
    }
    if (t->count == 2) {
        len += sprintf(buf + len, "%c", '@');
    }
    return print_string(buf, len, t);
}

size_t print_token(char* buf, size_t len, stoken_t *t) {
    len += sprintf(buf + len, "%c ", t->type);
    switch (t->type) {
    case 's':
        len = print_string(buf, len, t);
        break;
    case 'v':
        len = print_var(buf, len, t);
        break;
    default:
        len += sprintf(buf + len, "%s", t->val);
    }
    len += sprintf(buf + len, "%c", '\n');
    return len;
}

int read_file(const char* fname, int flags, int testtype)
{
    int count = 0;
    FILE *fp = NULL;
    char linebuf[4096];
    char g_actual[4096];
    char* bufptr;
    sfilter sf;
    int ok = 1;
    int num_tokens;
    int issqli;
    int i;

    g_test[0] = '\0';
    g_input[0] = '\0';
    g_expected[0] = '\0';

    fp = fopen(fname, "r");
    while(fgets(linebuf, sizeof(linebuf), fp) != NULL) {
        if (count == 0 && strcmp(linebuf, "--TEST--\n") == 0) {
            bufptr = g_test;
            count = 1;
        } else if (count == 1 && strcmp(linebuf, "--INPUT--\n") == 0) {
            bufptr = g_input;
            count = 2;
        } else if (count == 2 && strcmp(linebuf, "--EXPECTED--\n") == 0) {
            bufptr = g_expected;
            count = 3;
        } else {
            strcat(bufptr, linebuf);
        }
    }<commit_msg>get rid of gcc warning<commit_after>#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <glob.h>
#include "libinjection.h"

char g_test[4096];
char g_input[4096];
char g_expected[4096];

size_t modp_rtrim(char* str, size_t len)
{
    while (len) {
        char c = str[len -1];
        if (c == ' ' || c == '\n' || c == '\t' || c == '\r') {
            str[len -1] = '\0';
            len -= 1;
        } else {
            break;
        }
    }
    return len;
}

size_t print_string(char* buf, size_t len, stoken_t* t)
{
    /* print opening quote */
    if (t->str_open != '\0') {
        len += sprintf(buf + len, "%c", t->str_open);
    }

    /* print content */
    len += sprintf(buf + len, "%s", t->val);

    /* print closing quote */
    if (t->str_close != '\0') {
        len += sprintf(buf + len, "%c", t->str_close);
    }

    return len;
}

size_t print_var(char* buf, size_t len, stoken_t* t)
{
    if (t->count >= 1) {
        len += sprintf(buf + len, "%c", '@');
    }
    if (t->count == 2) {
        len += sprintf(buf + len, "%c", '@');
    }
    return print_string(buf, len, t);
}

size_t print_token(char* buf, size_t len, stoken_t *t) {
    len += sprintf(buf + len, "%c ", t->type);
    switch (t->type) {
    case 's':
        len = print_string(buf, len, t);
        break;
    case 'v':
        len = print_var(buf, len, t);
        break;
    default:
        len += sprintf(buf + len, "%s", t->val);
    }
    len += sprintf(buf + len, "%c", '\n');
    return len;
}

int read_file(const char* fname, int flags, int testtype)
{
    int count = 0;
    FILE *fp = NULL;
    char linebuf[4096];
    char g_actual[4096];
    char* bufptr = NULL;
    sfilter sf;
    int ok = 1;
    int num_tokens;
    int issqli;
    int i;

    g_test[0] = '\0';
    g_input[0] = '\0';
    g_expected[0] = '\0';

    fp = fopen(fname, "r");
    while(fgets(linebuf, sizeof(linebuf), fp) != NULL) {
        if (count == 0 && strcmp(linebuf, "--TEST--\n") == 0) {
            bufptr = g_test;
            count = 1;
        } else if (count == 1 && strcmp(linebuf, "--INPUT--\n") == 0) {
            bufptr = g_input;
            count = 2;
        } else if (count == 2 && strcmp(linebuf, "--EXPECTED--\n") == 0) {
            bufptr = g_expected;
            count = 3;
        } else {
            strcat(bufptr, linebuf);
        }
    }
{'fraction_non_alphanumeric': 0.19022429115531103, 'fraction_numerical': 0.019043588658484975, 'mean_word_length': 1.887599266951741, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1623069', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1787, 'n_words': 459}