<commit_before>/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "ioloop.h"
#include "restrict-access.h"
#include "auth-master.h"
#include "master-service.h"
#include "master-service-settings.h"
#include "notify-connection.h"
#include "doveadm-connection.h"
#include "replicator-brain.h"
#include "replicator-queue.h"
#include "replicator-settings.h"

#define REPLICATOR_AUTH_SERVICE_NAME "replicator"
#define REPLICATOR_DB_DUMP_INTERVAL_MSECS (1000*60*15)
/* if syncing fails, try again in 5 minutes */
#define REPLICATOR_FAILURE_RESYNC_INTERVAL_SECS (60*5)
#define REPLICATOR_DB_FNAME "replicator.db"

static struct replicator_queue *queue;
static struct replicator_brain *brain;
static const struct master_service_settings *service_set;
static const struct replicator_settings *set;
static struct timeout *to_dump;

static void client_connected(struct master_service_connection *conn)
{
	master_service_client_connection_accept(conn);
	if (strcmp(conn->name, "replicator-doveadm") == 0)
		doveadm_connection_create(queue, conn->fd);
	else
		(void)notify_connection_create(conn->fd, queue);
}

static void replication_add_users(struct replicator_queue *queue)
{
	struct auth_master_connection *auth_conn;
	struct auth_master_user_list_ctx *ctx;
	struct auth_user_info user_info;
	struct replicator_user *user;
	const char *path, *username;

	auth_conn = auth_master_init(set->auth_socket_path,
				     AUTH_MASTER_FLAG_NO_IDLE_TIMEOUT);

	memset(&user_info, 0, sizeof(user_info));
	user_info.service = REPLICATOR_AUTH_SERVICE_NAME;

	/* add all users into replication queue, so that we can start doing
	   full syncs for everyone whose state can't be found */
	ctx = auth_master_user_list_init(auth_conn, "", &user_info);
	while ((username = auth_master_user_list_next(ctx)) != NULL) {
		user = replicator_queue_add(queue, username,
					    REPLICATION_PRIORITY_NONE);
		user->last_update = 0;
	}
	if (auth_master_user_list_deinit(&ctx) < 0)
		i_error("listing users failed, can't replicate existing data");
	auth_master_deinit(&auth_conn);

	/* add updates from replicator db, if it exists */
	path = t_strconcat(service_set->state_dir, "/"REPLICATOR_DB_FNAME, NULL);
	(void)replicator_queue_import(queue, path);
}

static void ATTR_NULL(1)
replicator_dump_timeout(void *context ATTR_UNUSED)
{
	const char *path;

	path = t_strconcat(service_set->state_dir, "/"REPLICATOR_DB_FNAME, NULL);
	(void)replicator_queue_import(queue, path);
}
<commit_msg>replicator: Database should be exported (not imported!) every 15 mins.<commit_after>/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "ioloop.h"
#include "restrict-access.h"
#include "auth-master.h"
#include "master-service.h"
#include "master-service-settings.h"
#include "notify-connection.h"
#include "doveadm-connection.h"
#include "replicator-brain.h"
#include "replicator-queue.h"
#include "replicator-settings.h"

#define REPLICATOR_AUTH_SERVICE_NAME "replicator"
#define REPLICATOR_DB_DUMP_INTERVAL_MSECS (1000*60*15)
/* if syncing fails, try again in 5 minutes */
#define REPLICATOR_FAILURE_RESYNC_INTERVAL_SECS (60*5)
#define REPLICATOR_DB_FNAME "replicator.db"

static struct replicator_queue *queue;
static struct replicator_brain *brain;
static const struct master_service_settings *service_set;
static const struct replicator_settings *set;
static struct timeout *to_dump;

static void client_connected(struct master_service_connection *conn)
{
	master_service_client_connection_accept(conn);
	if (strcmp(conn->name, "replicator-doveadm") == 0)
		doveadm_connection_create(queue, conn->fd);
	else
		(void)notify_connection_create(conn->fd, queue);
}

static void replication_add_users(struct replicator_queue *queue)
{
	struct auth_master_connection *auth_conn;
	struct auth_master_user_list_ctx *ctx;
	struct auth_user_info user_info;
	struct replicator_user *user;
	const char *path, *username;

	auth_conn = auth_master_init(set->auth_socket_path,
				     AUTH_MASTER_FLAG_NO_IDLE_TIMEOUT);

	memset(&user_info, 0, sizeof(user_info));
	user_info.service = REPLICATOR_AUTH_SERVICE_NAME;

	/* add all users into replication queue, so that we can start doing
	   full syncs for everyone whose state can't be found */
	ctx = auth_master_user_list_init(auth_conn, "", &user_info);
	while ((username = auth_master_user_list_next(ctx)) != NULL) {
		user = replicator_queue_add(queue, username,
					    REPLICATION_PRIORITY_NONE);
		user->last_update = 0;
	}
	if (auth_master_user_list_deinit(&ctx) < 0)
		i_error("listing users failed, can't replicate existing data");
	auth_master_deinit(&auth_conn);

	/* add updates from replicator db, if it exists */
	path = t_strconcat(service_set->state_dir, "/"REPLICATOR_DB_FNAME, NULL);
	(void)replicator_queue_import(queue, path);
}

static void ATTR_NULL(1)
replicator_dump_timeout(void *context ATTR_UNUSED)
{
	const char *path;

	path = t_strconcat(service_set->state_dir, "/"REPLICATOR_DB_FNAME, NULL);
	(void)replicator_queue_export(queue, path);
}

{'fraction_non_alphanumeric': 0.10531515510768623, 'fraction_numerical': 0.008693934005137325, 'mean_word_length': 6.61203007518797, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3419268', 'n_tokens_mistral': 1853, 'n_tokens_neox': 1750, 'n_words': 395}