<commit_before>/*
 * libopensync - A synchronization framework
 * Copyright (C) 2004-2005  <NAME> <<EMAIL>>
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
 * 
 */
 
#include "opensync.h"
#include "opensync_internals.h"

/**
 * @ingroup OSyncConvPrivate
 */
/*@{*/

#ifndef DOXYGEN_SHOULD_SKIP_THIS
typedef struct conv_tree {
	OSyncFormatEnv *env;
	OSyncObjType *type;

	/* The converters that weren't reached yet */
	GList *unused;
	/* The search queue for the Breadth-first search */
	GList *search;
} conv_tree;

typedef struct vertice {
	OSyncObjFormat *format;
	
	/* The invoke_decap will return a new change everytime
	 * we run it so that the original change does not get
	 * changed. We also need to track if the data of this change
	 * should be freed or if it contains a reference into data of
	 * a previous change */
	OSyncChange *change;
	osync_bool free_change_data;
	osync_bool free_change;

	/** Keep reference counts because
	 * the data returned by converters
	 * can be references to other data,
	 * and we can't free them too early
	 */
	size_t references;

	/** The path of converters */
	GList *path;

	unsigned losses;
	unsigned objtype_changes;
	unsigned conversions;

} vertice;
#endif

static OSyncFormatConverter *osync_conv_find_converter_objformat(OSyncFormatEnv *env, OSyncObjFormat *fmt_src, OSyncObjFormat *fmt_trg)
{
	GList *element = NULL;
	for (element = env->converters; element; element = element->next) {
		OSyncFormatConverter *converter = element->data;
		if (fmt_src == converter->source_format && fmt_trg == converter->target_format)
			return converter;
	}
	return NULL;
}

osync_bool osync_converter_invoke(OSyncFormatConverter *converter, OSyncChange *change, void *converter_data, OSyncError **error)
{
	osync_trace(TRACE_ENTRY, "osync_converter_invoke(%p, %p, %p)", converter, change, error);
	osync_trace(TRACE_INTERNAL, "converter: Type: %i, source: %s, target %s", converter->type, converter->source_format->name, converter->target_format->name);
	char *data = NULL;
	int datasize = 0;
	osync_bool ret = TRUE;
	if (converter->type == CONVERTER_DETECTOR && !converter->convert_func) {
		change->format = converter->target_format;
		change->objtype = osync_change_get_objformat(change)->objtype;
		osync_trace(TRACE_EXIT, "osync_converter_invoke: TRUE: Detector path");
		return TRUE;
	}
	
	if (!converter->convert_func) {
		osync_error_set(error, OSYNC_ERROR_GENERIC, "Invalid converter");
		osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke: %s", osync_error_print(error));
		return FALSE;
	}
	
	if (change->data) {
		//Invoke the converter and all extensions
		//osync_conv_invoke_extensions(converter->source_format, FALSE, change);
		osync_bool free_input = FALSE;
		if ((ret = converter->convert_func(converter_data, change->data, change->size, &data, &datasize, &free_input, error))) {
		
			if (converter->type == CONVERTER_DECAP) {
				if (!free_input) {
					/* Duplicate the returned data, as the original data will be destroyed */
					if (!converter->target_format->copy_func) {
						/* There is nothing we can do, here. The returned data is a reference, but
						 * we can't copy the data before destroying it
						 */
						osync_debug("OSYNC", 0, "Format %s don't have a copy function, but a no-copy converter was registered", converter->target_format->name);
						osync_error_set(error, OSYNC_ERROR_GENERIC, "Format %s don't have a copy function, but a no-copy converter was registered", converter->target_format->name);
						osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke: %s", osync_error_print(error));
						return FALSE;
					}
					converter->target_format->copy_func(data, datasize, &data, &datasize);
				}
			}
			/* Free the data, unless the converter took the ownership of the data */
			if (free_input) {
				if (converter->source_format->destroy_func) {
					converter->source_format->destroy_func(change->data, change->size);
				} else
					osync_debug("OSYNC", 1, "Format %s don't have a destroy function. Possible memory leak", converter->source_format->name);
			}
			change->data = data;
			change->size = datasize;
			
			//osync_conv_invoke_extensions(converter->target_format, TRUE, change);
		}
	}
	
	if (ret) {
		osync_debug("OSYNC", 3, "Converting! replacing format %s with %s", converter->source_format->name, converter->target_format->name);
		change->format = converter->target_format;
		change->objtype = osync_change_get_objformat(change)->objtype;
		osync_trace(TRACE_EXIT, "osync_converter_invoke: TRUE");
	} else
		osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke: %s", osync_error_print(error));
	return ret;
}

OSyncChange *osync_converter_invoke_decap(OSyncFormatConverter *converter, OSyncChange *change, osync_bool *free_output)
{
	osync_trace(TRACE_ENTRY, "osync_converter_invoke_decap(%p, %p, %p)", converter, change, free_output);
	
	*free_output = FALSE;
	
	if (!converter->convert_func) {
		osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke_decap: No convert function");
		return NULL;
	}
	
	if (converter->type != CONVERTER_DECAP) {
		osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke_decap: Not a decap");
		return NULL;
	}
	
	OSyncChange *new_change = osync_change_new();
	
	
	if (change->data) {
		//Invoke the converter and all extensions
		OSyncError *error = NULL;
		if (!converter->convert_func(NULL, change->data, change->size, &(new_change->data), &(new_change->size), free_output, &error)) {
			osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke_decap: %s", osync_error_print(&error));
			osync_error_free(&error);
			return NULL;
		}
		new_change->has_data = change->has_data;
	}
	osync_debug("OSYNC", 3, "Converting! replacing format %s with %s", converter->source_format->name, converter->target_format->name);
	new_change->format = converter->target_format;
	new_change->objtype = osync_change_get_objformat(new_change)->objtype;
	osync_trace(TRACE_EXIT, "osync_converter_invoke_decap: %p", new_change);
	return new_change;
}

/** Compare the distance of two vertices
 *
 * First, try to minimize the losses. Then,
 * try to minimize the conversions between
 * different objtypes. Then, try to minimize
 * the total number of conversions.
 */
int compare_vertice_distance(const void *a, const void *b)
{
	const vertice *va = a;
	const vertice *vb = b;
	if (va->losses < vb->losses)
		return -1;
	else if (va->losses > vb->losses)
		return 1;
	else if (va->objtype_changes < vb->objtype_changes)
		return -1;
	else if (va->objtype_changes > vb->objtype_changes)
		return 1;
	else if (va->conversions < vb->conversions)
		return -1;
	else if (va->conversions > vb->conversions)
		return 1;
	else
		return 0;
}

/** Increment a vertice reference count */
/*static void ref_vertice(vertice *v)
{
	v->references++;
}*/

/** Dereference an vertice
 */
static void deref_vertice(vertice *vertice)
{
	/* Decrement the reference count,
	 * and just return if we still
	 * have a reference
	 */
	if (--vertice->references > 0)
		return;

	g_list_free(vertice->path);
	if (vertice->change && vertice->free_change) {
		if (vertice->free_change_data)
			osync_change_free_data(vertice->change);
		osync_change_free(vertice->change);
	}

	g_free(vertice);
}

/** Returns a neighbour of the vertice ve
 *
 * Returns a new reference to te vertice. The reference
 * should be dropped using deref_vertice(), later.
 */
vertice *get_next_vertice_neighbour(OSyncFormatEnv *env, conv_tree *tree, vertice *ve)
{
	GList *c = NULL;
	osync_trace(TRACE_ENTRY, "get_next_vertice_neighbour(%p, %p, %p:%s)", env, tree, ve, ve->format ? ve->format->name : "None");
		
	for (c = tree->unused; c; c = c->next) {
		OSyncFormatConverter *converter = c->data;
		OSyncObjFormat *fmt_target = converter->target_format;
		
		/* Check only valid converters, from the right format */
		if (strcmp(converter->source_format->name, ve->format->name))
			continue;

		// If the converter type is a detector we need to know wether the input is correct
		if (converter->detect_func) {
			if (!converter->detect_func(env, ve->change->data, ve->change->size)) {
				osync_trace(TRACE_INTERNAL, "Invoked detector for converter from %s to %s: FALSE", converter->source_format->name, converter->target_format->name);
				continue;
			}
			osync_trace(TRACE_INTERNAL, "Invoked detector for converter from %s to %s: TRUE", converter->source_format->name, converter->target_format->name);
		}

		OSyncChange *new_change = NULL;
		osync_bool free_output = TRUE;
		if (converter->type == CONVERTER_DECAP)
			if (!(new_change = osync_converter_invoke_decap(converter, ve->change, &free_output)))
				continue;

		/* From this point, we already found an edge (i.e. a converter) that may
		 * be used
		 */

		/* Remove the converter from the unused list */
		tree->unused = g_list_remove(tree->unused, converter);

		/* Allocate the new neighbour */
		vertice *neigh = g_malloc0(sizeof(vertice));
		/* Start with a reference count = 1 */
		neigh->references = 1;
		neigh->format = fmt_target;
		neigh->path = g_list_copy(ve->path);
		neigh->path = g_list_append(neigh->path, converter);

		if (new_change) {
			neigh->change = new_change;
			neigh->free_change = TRUE;
			neigh->free_change_data = free_output;
		} else {
			neigh->change = ve->change;
			neigh->free_change = FALSE;
			neigh->free_change_data = FALSE;
		}
		<commit_msg>Fix on conversion code: don't fill the change field on vertices, if a conversion wasn't done and we don't have the converted data for the vertice<commit_after>/*
 * libopensync - A synchronization framework
 * Copyright (C) 2004-2005  <NAME> <<EMAIL>>
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
 * 
 */
 
#include "opensync.h"
#include "opensync_internals.h"

/**
 * @ingroup OSyncConvPrivate
 */
/*@{*/

#ifndef DOXYGEN_SHOULD_SKIP_THIS
typedef struct conv_tree {
	OSyncFormatEnv *env;
	OSyncObjType *type;

	/* The converters that weren't reached yet */
	GList *unused;
	/* The search queue for the Breadth-first search */
	GList *search;
} conv_tree;

typedef struct vertice {
	OSyncObjFormat *format;
	
	/* The invoke_decap will return a new change everytime
	 * we run it so that the original change does not get
	 * changed. We also need to track if the data of this change
	 * should be freed or if it contains a reference into data of
	 * a previous change */
	OSyncChange *change;
	osync_bool free_change_data;
	osync_bool free_change;

	/** Keep reference counts because
	 * the data returned by converters
	 * can be references to other data,
	 * and we can't free them too early
	 */
	size_t references;

	/** The path of converters */
	GList *path;

	unsigned losses;
	unsigned objtype_changes;
	unsigned conversions;

} vertice;
#endif

static OSyncFormatConverter *osync_conv_find_converter_objformat(OSyncFormatEnv *env, OSyncObjFormat *fmt_src, OSyncObjFormat *fmt_trg)
{
	GList *element = NULL;
	for (element = env->converters; element; element = element->next) {
		OSyncFormatConverter *converter = element->data;
		if (fmt_src == converter->source_format && fmt_trg == converter->target_format)
			return converter;
	}
	return NULL;
}

osync_bool osync_converter_invoke(OSyncFormatConverter *converter, OSyncChange *change, void *converter_data, OSyncError **error)
{
	osync_trace(TRACE_ENTRY, "osync_converter_invoke(%p, %p, %p)", converter, change, error);
	osync_trace(TRACE_INTERNAL, "converter: Type: %i, source: %s, target %s", converter->type, converter->source_format->name, converter->target_format->name);
	char *data = NULL;
	int datasize = 0;
	osync_bool ret = TRUE;
	if (converter->type == CONVERTER_DETECTOR && !converter->convert_func) {
		change->format = converter->target_format;
		change->objtype = osync_change_get_objformat(change)->objtype;
		osync_trace(TRACE_EXIT, "osync_converter_invoke: TRUE: Detector path");
		return TRUE;
	}
	
	if (!converter->convert_func) {
		osync_error_set(error, OSYNC_ERROR_GENERIC, "Invalid converter");
		osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke: %s", osync_error_print(error));
		return FALSE;
	}
	
	if (change->data) {
		//Invoke the converter and all extensions
		//osync_conv_invoke_extensions(converter->source_format, FALSE, change);
		osync_bool free_input = FALSE;
		if ((ret = converter->convert_func(converter_data, change->data, change->size, &data, &datasize, &free_input, error))) {
		
			if (converter->type == CONVERTER_DECAP) {
				if (!free_input) {
					/* Duplicate the returned data, as the original data will be destroyed */
					if (!converter->target_format->copy_func) {
						/* There is nothing we can do, here. The returned data is a reference, but
						 * we can't copy the data before destroying it
						 */
						osync_debug("OSYNC", 0, "Format %s don't have a copy function, but a no-copy converter was registered", converter->target_format->name);
						osync_error_set(error, OSYNC_ERROR_GENERIC, "Format %s don't have a copy function, but a no-copy converter was registered", converter->target_format->name);
						osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke: %s", osync_error_print(error));
						return FALSE;
					}
					converter->target_format->copy_func(data, datasize, &data, &datasize);
				}
			}
			/* Free the data, unless the converter took the ownership of the data */
			if (free_input) {
				if (converter->source_format->destroy_func) {
					converter->source_format->destroy_func(change->data, change->size);
				} else
					osync_debug("OSYNC", 1, "Format %s don't have a destroy function. Possible memory leak", converter->source_format->name);
			}
			change->data = data;
			change->size = datasize;
			
			//osync_conv_invoke_extensions(converter->target_format, TRUE, change);
		}
	}
	
	if (ret) {
		osync_debug("OSYNC", 3, "Converting! replacing format %s with %s", converter->source_format->name, converter->target_format->name);
		change->format = converter->target_format;
		change->objtype = osync_change_get_objformat(change)->objtype;
		osync_trace(TRACE_EXIT, "osync_converter_invoke: TRUE");
	} else
		osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke: %s", osync_error_print(error));
	return ret;
}

OSyncChange *osync_converter_invoke_decap(OSyncFormatConverter *converter, OSyncChange *change, osync_bool *free_output)
{
	osync_trace(TRACE_ENTRY, "osync_converter_invoke_decap(%p, %p, %p)", converter, change, free_output);
	
	*free_output = FALSE;
	
	if (!converter->convert_func) {
		osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke_decap: No convert function");
		return NULL;
	}
	
	if (converter->type != CONVERTER_DECAP) {
		osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke_decap: Not a decap");
		return NULL;
	}
	
	OSyncChange *new_change = osync_change_new();
	
	
	if (change->data) {
		//Invoke the converter and all extensions
		OSyncError *error = NULL;
		if (!converter->convert_func(NULL, change->data, change->size, &(new_change->data), &(new_change->size), free_output, &error)) {
			osync_trace(TRACE_EXIT_ERROR, "osync_converter_invoke_decap: %s", osync_error_print(&error));
			osync_error_free(&error);
			return NULL;
		}
		new_change->has_data = change->has_data;
	}
	osync_debug("OSYNC", 3, "Converting! replacing format %s with %s", converter->source_format->name, converter->target_format->name);
	new_change->format = converter->target_format;
	new_change->objtype = osync_change_get_objformat(new_change)->objtype;
	osync_trace(TRACE_EXIT, "osync_converter_invoke_decap: %p", new_change);
	return new_change;
}

/** Compare the distance of two vertices
 *
 * First, try to minimize the losses. Then,
 * try to minimize the conversions between
 * different objtypes. Then, try to minimize
 * the total number of conversions.
 */
int compare_vertice_distance(const void *a, const void *b)
{
	const vertice *va = a;
	const vertice *vb = b;
	if (va->losses < vb->losses)
		return -1;
	else if (va->losses > vb->losses)
		return 1;
	else if (va->objtype_changes < vb->objtype_changes)
		return -1;
	else if (va->objtype_changes > vb->objtype_changes)
		return 1;
	else if (va->conversions < vb->conversions)
		return -1;
	else if (va->conversions > vb->conversions)
		return 1;
	else
		return 0;
}

/** Increment a vertice reference count */
/*static void ref_vertice(vertice *v)
{
	v->references++;
}*/

/** Dereference an vertice
 */
static void deref_vertice(vertice *vertice)
{
	/* Decrement the reference count,
	 * and just return if we still
	 * have a reference
	 */
	if (--vertice->references > 0)
		return;

	g_list_free(vertice->path);
	if (vertice->change && vertice->free_change) {
		if (vertice->free_change_data)
			osync_change_free_data(vertice->change);
		osync_change_free(vertice->change);
	}

	g_free(vertice);
}

/** Returns a neighbour of the vertice ve
 *
 * Returns a new reference to te vertice. The reference
 * should be dropped using deref_vertice(), later.
 */
vertice *get_next_vertice_neighbour(OSyncFormatEnv *env, conv_tree *tree, vertice *ve)
{
	GList *c = NULL;
	osync_trace(TRACE_ENTRY, "get_next_vertice_neighbour(%p, %p, %p:%s)", env, tree, ve, ve->format ? ve->format->name : "None");
		
	for (c = tree->unused; c; c = c->next) {
		OSyncFormatConverter *converter = c->data;
		OSyncObjFormat *fmt_target = converter->target_format;
		
		/* Check only valid converters, from the right format */
		if (strcmp(converter->source_format->name, ve->format->name))
			continue;

		// If the converter type is a detector we need to know wether the input is correct
		if (converter->detect_func) {
			if (!ve->change) {
				osync_trace(TRACE_INTERNAL,
						"We would call a converter to %s, but there is no change data on vertice", fmt_target->name);
				continue;
			}
			if (!converter->detect_func(env, ve->change->data, ve->change->size)) {
				osync_trace(TRACE_INTERNAL, "Invoked detector for converter from %s to %s: FALSE", converter->source_format->name, converter->target_format->name);
				continue;
			}
			osync_trace(TRACE_INTERNAL, "Invoked detector for converter from %s to %s: TRUE", converter->source_format->name, converter->target_format->name);
		}

		OSyncChange *new_change = NULL;
		osync_bool free_output = TRUE;
		if (converter->type == CONVERTER_DECAP) {
			if (!ve->change) {
				osync_trace(TRACE_INTERNAL, "A desencapsulator to %s would be called, but we can't because the data on this vertice wasn't converted", fmt_target->name);
				continue;
			}
			if (!(new_change = osync_converter_invoke_decap(converter, ve->change, &free_output)))
				continue;
		}

		/* From this point, we already found an edge (i.e. a converter) that may
		 * be used
		 */

		/* Remove the converter from the unused list */
		tree->unused = g_list_remove(tree->unused, converter);

		/* Allocate the new neighbour */
		vertice *neigh = g_malloc0(sizeof(vertice));
		/* Start with a reference count = 1 */
		neigh->references = 1;
		neigh->format = fmt_target;
		neigh->path = g_list_copy(ve->path);
		neigh->path = g_list_append(neigh->path, converter);

		if (new_change) {
			neigh->change = new_change;
			neigh->free_change = TRUE;
			neigh->free_change_data = free_output;
		} else {
			neigh->change = NULL;
			neigh->free_change = FALSE;
			neigh->free_change_data = FALSE;
		}
		
{'fraction_non_alphanumeric': 0.11678015564202335, 'fraction_numerical': 0.0038910505836575876, 'mean_word_length': 4.474174653887114, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 280, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1976464', 'n_tokens_mistral': 7378, 'n_tokens_neox': 6680, 'n_words': 1904}