<commit_before>/*
 * make sure _GNU_SOURCE is defined
 */
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif

#include <sys/types.h>
#include <sys/stat.h>

#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
#include <fcntl.h>
#include <assert.h>

#include "log.h"
#include "stat_cache.h"
#include "fdevent.h"
#include "etag.h"

#ifdef HAVE_ATTR_ATTRIBUTES_H
#include <attr/attributes.h>
#endif

#include "sys-mmap.h"
#include "sys-files.h"
#include "sys-strings.h"

#undef HAVE_FAM_H
#undef HAVE_SYS_INOTIFY_H

#include <glib.h>
#include <pthread.h>

#if 0
/* enables debug code for testing if all nodes in the stat-cache as accessable */
#define DEBUG_STAT_CACHE
#endif

/*
 * stat-cache
 *
 * we cache the stat() calls in our own storage
 * the directories are cached in FAM
 *
 * if we get a change-event from FAM, we increment the version in the FAM->dir mapping
 *
 * if the stat()-cache is queried we check if the version id for the directory is the
 * same and return immediatly.
 *
 *
 * What we need:
 *
 * - for each stat-cache entry we need a fast indirect lookup on the directory name
 * - for each FAMRequest we have to find the version in the directory cache (index as userdata)
 *
 * stat <<-> directory <-> FAMRequest
 *
 * if file is deleted, directory is dirty, file is rechecked ...
 * if directory is deleted, directory mapping is removed
 *
 * */

/* the directory name is too long to always compare on it
 * - we need a hash
 * - the hash-key is used as sorting criteria for a tree
 * - a splay-tree is used as we can use the caching effect of it
 */

/* we want to cleanup the stat-cache every few seconds, let's say 10
 *
 * - remove entries which are outdated since 30s
 * - remove entries which are fresh but havn't been used since 60s
 * - if we don't have a stat-cache entry for a directory, release it from the monitor
 */

typedef struct { 
	buffer *name;

	void *con;
} stat_job;

stat_job *stat_job_init() {
	stat_job *sj = calloc(1, sizeof(*sj));

	sj->name = buffer_init();
	
	return sj;
}

void stat_job_free(stat_job *sj) {
	if (!sj) return;

	buffer_free(sj->name);

	free(sj);
}

gpointer stat_cache_thread(gpointer _srv) {
	server *srv = (server *)_srv;
	stat_job *sj = NULL;
	
	/* take the stat-job-queue */
	GAsyncQueue * inq = g_async_queue_ref(srv->stat_queue);
	GAsyncQueue * outq = g_async_queue_ref(srv->joblist_queue);

	/* */
	while (!srv->is_shutdown) {
		/* let's see what we have to stat */
		struct stat st;
		GTimeVal ts;

		/* wait one second as the poll() */<commit_msg>let's try to compile on glib2 2.4.0<commit_after>/*
 * make sure _GNU_SOURCE is defined
 */
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif

#include <sys/types.h>
#include <sys/stat.h>

#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
#include <fcntl.h>
#include <assert.h>

#include <glib.h>

#include "log.h"
#include "stat_cache.h"
#include "fdevent.h"
#include "etag.h"

#ifdef HAVE_ATTR_ATTRIBUTES_H
#include <attr/attributes.h>
#endif

#include "sys-mmap.h"
#include "sys-files.h"
#include "sys-strings.h"

#undef HAVE_FAM_H
#undef HAVE_SYS_INOTIFY_H

#if 0
/* enables debug code for testing if all nodes in the stat-cache as accessable */
#define DEBUG_STAT_CACHE
#endif

/*
 * stat-cache
 *
 * we cache the stat() calls in our own storage
 * the directories are cached in FAM
 *
 * if we get a change-event from FAM, we increment the version in the FAM->dir mapping
 *
 * if the stat()-cache is queried we check if the version id for the directory is the
 * same and return immediatly.
 *
 *
 * What we need:
 *
 * - for each stat-cache entry we need a fast indirect lookup on the directory name
 * - for each FAMRequest we have to find the version in the directory cache (index as userdata)
 *
 * stat <<-> directory <-> FAMRequest
 *
 * if file is deleted, directory is dirty, file is rechecked ...
 * if directory is deleted, directory mapping is removed
 *
 * */

/* the directory name is too long to always compare on it
 * - we need a hash
 * - the hash-key is used as sorting criteria for a tree
 * - a splay-tree is used as we can use the caching effect of it
 */

/* we want to cleanup the stat-cache every few seconds, let's say 10
 *
 * - remove entries which are outdated since 30s
 * - remove entries which are fresh but havn't been used since 60s
 * - if we don't have a stat-cache entry for a directory, release it from the monitor
 */

typedef struct { 
	buffer *name;

	void *con;
} stat_job;

stat_job *stat_job_init() {
	stat_job *sj = calloc(1, sizeof(*sj));

	sj->name = buffer_init();
	
	return sj;
}

void stat_job_free(stat_job *sj) {
	if (!sj) return;

	buffer_free(sj->name);

	free(sj);
}

gpointer stat_cache_thread(gpointer _srv) {
	server *srv = (server *)_srv;
	stat_job *sj = NULL;
	
	/* take the stat-job-queue */
	GAsyncQueue * inq; 
	GAsyncQueue * outq;

	g_async_queue_ref(srv->joblist_queue);
	g_async_queue_ref(srv->stat_queue);

	inq = srv->stat_queue;
	outq = srv->joblist_queue;

	/* */
	while (!srv->is_shutdown) {
		/* let's see what we have to stat */
		struct stat st;
		GTimeVal ts;

		/* wait one second as the poll() */
{'fraction_non_alphanumeric': 0.11277173913043478, 'fraction_numerical': 0.0038819875776397515, 'mean_word_length': 3.5886019590382903, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7010920', 'n_tokens_mistral': 1869, 'n_tokens_neox': 1783, 'n_words': 656}