<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <mysql.h>

#include "clbuf.h"
#include "rtgconf.h"
#include "multithread.h"
#include "cllog.h"
#include "database.h"
#include "globals.h"

#define COMMIT_INTERVAL 100

MYSQL *connection(struct rtgconf *config);

void *database_run(void *ptr)
{
        struct mt_context *thread_context = (struct mt_context *) ptr;
        struct database_ctx *database_context = (struct database_ctx *) thread_context->param;
        unsigned my_id = thread_context->thread_id;
        struct rtgconf *config = database_context->config;

        unsigned useless_iterations = 0;
        unsigned query_counter = 0;

        MYSQL *conn = 0;

        while (!thread_stop_requested) {
                if (use_db) {
                        if (clbuf_count_used(queries) > 0) {
                                if (conn == 0 || mysql_ping(conn) != 0) {
                                        if (conn)
                                                mysql_close(conn);
                                        cllog(2, "DB thread %d connecting to MySQL.", my_id);
                                        conn = connection(config);
                                        if (conn)
                                                cllog(2, "DB thread %d is connected to MySQL.", my_id);
                                } else {
                                        useless_iterations = 0;
                                        query_counter++;

                                        if (query_counter % COMMIT_INTERVAL == 0) {
                                                mysql_commit(conn);
                                                cllog(2, "DB thread %d committed transaction due to configured commit interval (query_counter = %u).", my_id, query_counter);
                                        }

                                        char *query = (char *) clbuf_pop(queries);
                                        if (query) {
                                                int result = mysql_query(conn, query);
                                                cllog(3, "DB thread %d executed query with result %d.", my_id, result);
                                                free(query);
                                        } else {
                                                cllog(2, "DB thread %d dequeued NULL query.", my_id);
                                        }
                                }
                        } else {
                                useless_iterations++;

                                if (useless_iterations > 5) {
                                        mysql_commit(conn);
                                        cllog(2, "DB thread %d committed transaction at end of run (query_counter = %u).", my_id, query_counter);
                                        query_counter = 0;
                                }

                                if (useless_iterations > 15 && conn != 0 && mysql_ping(conn)) {
                                        cllog(2, "DB thread %d disconnecting.", my_id);
                                        mysql_close(conn);
                                        conn = 0;
                                }

                                sleep(1);
                        }
                } else {
                        char *query = (char *) clbuf_pop(queries);
                        if (query) {
                                cllog(3, "%s", query);
                                free(query);
                        }
                        if (clbuf_count_used(queries) == 0)
                                sleep(1);
                }
        }

        if (conn) {
                cllog(1, "DB thread %d shutting down.", my_id);
                mysql_commit(conn);
                mysql_close(conn);
        }

        return 0;
}

MYSQL *connection(struct rtgconf *config)
{
        MYSQL *conn = mysql_init(NULL);
        if (conn == NULL) {
                cllog(0, "Error %u: %s\n", mysql_errno(conn), mysql_error(conn));
                return NULL;
        }
        if (mysql_real_connect(conn, config->dbhost, config->dbuser, config->dbpass, config->database, 0, NULL, 0) == NULL) {
                cllog(0, "Error %u: %s\n", mysql_errno(conn), mysql_error(conn));
                return NULL;
        }
        mysql_autocommit(conn, 0);

        return conn;
}<commit_msg>Don't get stuck in a tight loop if we can't connect to the DB.<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <mysql.h>

#include "clbuf.h"
#include "rtgconf.h"
#include "multithread.h"
#include "cllog.h"
#include "database.h"
#include "globals.h"

#define COMMIT_INTERVAL 100

MYSQL *connection(struct rtgconf *config);

void *database_run(void *ptr)
{
        struct mt_context *thread_context = (struct mt_context *) ptr;
        struct database_ctx *database_context = (struct database_ctx *) thread_context->param;
        unsigned my_id = thread_context->thread_id;
        struct rtgconf *config = database_context->config;

        unsigned useless_iterations = 0;
        unsigned query_counter = 0;

        MYSQL *conn = 0;

        while (!thread_stop_requested) {
                if (use_db) {
                        if (clbuf_count_used(queries) > 0) {
                                if (conn == 0 || mysql_ping(conn) != 0) {
                                        if (conn)
                                                mysql_close(conn);
                                        cllog(2, "DB thread %d connecting to MySQL.", my_id);
                                        conn = connection(config);
                                        if (conn)
                                                cllog(2, "DB thread %d is connected to MySQL.", my_id);
                                        else
                                                sleep(1); /* Don't get stuck in a tight loop if we can't connect. */
                                } else {
                                        useless_iterations = 0;
                                        query_counter++;

                                        if (query_counter % COMMIT_INTERVAL == 0) {
                                                mysql_commit(conn);
                                                cllog(2, "DB thread %d committed transaction due to configured commit interval (query_counter = %u).", my_id, query_counter);
                                        }

                                        char *query = (char *) clbuf_pop(queries);
                                        if (query) {
                                                int result = mysql_query(conn, query);
                                                cllog(3, "DB thread %d executed query with result %d.", my_id, result);
                                                free(query);
                                        } else {
                                                cllog(2, "DB thread %d dequeued NULL query.", my_id);
                                        }
                                }
                        } else {
                                useless_iterations++;

                                if (useless_iterations > 5) {
                                        mysql_commit(conn);
                                        cllog(2, "DB thread %d committed transaction at end of run (query_counter = %u).", my_id, query_counter);
                                        query_counter = 0;
                                }

                                if (useless_iterations > 15 && conn != 0 && mysql_ping(conn)) {
                                        cllog(2, "DB thread %d disconnecting.", my_id);
                                        mysql_close(conn);
                                        conn = 0;
                                }

                                sleep(1);
                        }
                } else {
                        char *query = (char *) clbuf_pop(queries);
                        if (query) {
                                cllog(3, "%s", query);
                                free(query);
                        }
                        if (clbuf_count_used(queries) == 0)
                                sleep(1);
                }
        }

        if (conn) {
                cllog(1, "DB thread %d shutting down.", my_id);
                mysql_commit(conn);
                mysql_close(conn);
        }

        return 0;
}

MYSQL *connection(struct rtgconf *config)
{
        MYSQL *conn = mysql_init(NULL);
        if (conn == NULL) {
                cllog(0, "MySQL error %u: %s", mysql_errno(conn), mysql_error(conn));
                return NULL;
        }
        if (mysql_real_connect(conn, config->dbhost, config->dbuser, config->dbpass, config->database, 0, NULL, 0) == NULL) {
                cllog(0, "MySQL error %u: %s", mysql_errno(conn), mysql_error(conn));
                return NULL;
        }
        mysql_autocommit(conn, 0);

        return conn;
}
{'fraction_non_alphanumeric': 0.08989865969271003, 'fraction_numerical': 0.007736733137190803, 'mean_word_length': 0.9148758606300855, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 95, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1403385', 'n_tokens_mistral': 2143, 'n_tokens_neox': 2011, 'n_words': 538}