<commit_before>/*
 * consumer_xml.c -- a libxml2 serialiser of mlt service networks
 * Copyright (C) 2003-2018 Meltytech, LLC
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "common.h"

#include <framework/mlt.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <locale.h>
#include <libxml/tree.h>
#include <pthread.h>

#define ID_SIZE 128
#define TIME_PROPERTY "_consumer_xml"

#define _x (const xmlChar*)
#define _s (const char*)

// This maintains counters for adding ids to elements
struct serialise_context_s
{
	mlt_properties id_map;
	int producer_count;
	int multitrack_count;
	int playlist_count;
	int tractor_count;
	int filter_count;
	int transition_count;
	int pass;
	mlt_properties hide_map;
	char *root;
	char *store;
	int no_meta;
	mlt_profile profile;
	mlt_time_format time_format;
};
typedef struct serialise_context_s* serialise_context;

/** Forward references to static functions.
*/

static int consumer_start( mlt_consumer parent );
static int consumer_stop( mlt_consumer parent );
static int consumer_is_stopped( mlt_consumer consumer );
static void *consumer_thread( void *arg );
static void serialise_service( serialise_context context, mlt_service service, xmlNode *node );

typedef enum
{
	xml_existing,
	xml_producer,
	xml_multitrack,
	xml_playlist,
	xml_tractor,
	xml_filter,
	xml_transition
}
xml_type;

/** Create or retrieve an id associated to this service.
*/

static char *xml_get_id( serialise_context context, mlt_service service, xml_type type )
{
	char *id = NULL;
	int i = 0;
	mlt_properties map = context->id_map;

	// Search the map for the service
	for ( i = 0; i < mlt_properties_count( map ); i ++ )
		if ( mlt_properties_get_data_at( map, i, NULL ) == service )
			break;

	// If the service is not in the map, and the type indicates a new id is needed...
	if ( i >= mlt_properties_count( map ) && type != xml_existing )
	{
		// Attempt to reuse existing id
		id = mlt_properties_get( MLT_SERVICE_PROPERTIES( service ), "id" );

		// If no id, or the id is used in the map (for another service), then
		// create a new one.
		if ( id == NULL || mlt_properties_get_data( map, id, NULL ) != NULL )
		{
			char temp[ ID_SIZE ];
			do
			{
				switch( type )
				{
					case xml_producer:
						sprintf( temp, "producer%d", context->producer_count ++ );
						break;
					case xml_multitrack:
						sprintf( temp, "multitrack%d", context->multitrack_count ++ );
						break;
					case xml_playlist:
						sprintf( temp, "playlist%d", context->playlist_count ++ );
						break;
					case xml_tractor:
						sprintf( temp, "tractor%d", context->tractor_count ++ );
						break;
					case xml_filter:
						sprintf( temp, "filter%d", context->filter_count ++ );
						break;
					case xml_transition:
						sprintf( temp, "transition%d", context->transition_count ++ );
						break;
					case xml_existing:
						// Never gets here
						break;
				}
			}
			while( mlt_properties_get_data( map, temp, NULL ) != NULL );

			// Set the data at the generated name
			mlt_properties_set_data( map, temp, service, 0, NULL, NULL );

			// Get the pointer to the name (i is the end of the list)
			id = mlt_properties_get_name( map, i );
		}
		else
		{
			// Store the existing id in the map
			mlt_properties_set_data( map, id, service, 0, NULL, NULL );
		}
	}
	else if ( type == xml_existing )
	{
		id = mlt_properties_get_name( map, i );
	}

	return id;
}

/** This is what will be called by the factory - anything can be passed in
	via the argument, but keep it simple.
*/

mlt_consumer consumer_xml_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
{
	// Create the consumer object
	mlt_consumer consumer = calloc( 1, sizeof( struct mlt_consumer_s ) );

	// If no malloc'd and consumer init ok
	if ( consumer != NULL && mlt_consumer_init( consumer, NULL, profile ) == 0 )
	{
		// Allow thread to be started/stopped
		consumer->start = consumer_start;
		consumer->stop = consumer_stop;
		consumer->is_stopped = consumer_is_stopped;

		mlt_properties_set( MLT_CONSUMER_PROPERTIES( consumer ), "resource", arg );
		mlt_properties_set_int( MLT_CONSUMER_PROPERTIES( consumer ), "real_time", 0 );
		mlt_properties_set_int( MLT_CONSUMER_PROPERTIES( consumer ), "prefill", 1 );
		mlt_properties_set_int( MLT_CONSUMER_PROPERTIES( consumer ), "terminate_on_pause", 1 );

		// Return the consumer produced
		return consumer;
	}

	// malloc or consumer init failed
	free( consumer );

	// Indicate failure
	return NULL;
}

static void serialise_properties( serialise_context context, mlt_properties properties, xmlNode *node )
{
	int i;
	xmlNode *p;

	// Enumerate the properties
	for ( i = 0; i < mlt_properties_count( properties ); i++ )
	{
		char *name = mlt_properties_get_name( properties, i );
		if ( name != NULL &&
			 name[ 0 ] != '_' &&
			 mlt_properties_get_value( properties, i ) != NULL &&
			 ( !context->no_meta || strncmp( name, "meta.", 5 ) ) &&
			 strcmp( name, "mlt" ) &&
			 strcmp( name, "mlt_type" ) &&
			 strcmp( name, "in" ) &&
			 strcmp( name, "out" ) &&
			 strcmp( name, "id" ) &&
			 strcmp( name, "title" ) &&
			 strcmp( name, "root" ) &&
			 strcmp( name, "width" ) &&
			 strcmp( name, "height" ) )
		{
			// Force a string with keyframes to be parsed with mlt_animation to
			// facilitate output with the specified time format.
			mlt_properties_anim_get_int( properties, name, 0, 0 );

			char *value = mlt_properties_get_value_tf( properties, i, context->time_format );
			if ( value )
			{
				int rootlen = strlen( context->root );
				const char *value_orig = value;
				size_t prefix_size = mlt_xml_prefix_size( properties, name, value );

				// Strip off prefix.
				if ( prefix_size )
					value += prefix_size;

				// Ignore trailing slash on root.
				if ( rootlen && ( context->root[rootlen - 1] == '/' || context->root[rootlen - 1] == '\\') )
					--rootlen;

				// convert absolute path to relative
				if ( rootlen && !strncmp( value, context->root, rootlen ) &&
					( value[rootlen] == '/' || value[rootlen] == '\\' ) )
				{
					if ( prefix_size )
					{
						char *s = calloc( 1, strlen( value_orig ) - rootlen + 1 );
						strncat( s, value_orig, prefix_size );
						strcat( s, value + rootlen + 1 );
						p = xmlNewTextChild( node, NULL, _x("property"), _x(s) );
						free( s );
					} else {
						p = xmlNewTextChild( node, NULL, _x("property"), _x(value_orig + rootlen + 1) );
					}
				}
				else
					p = xmlNewTextChild( node, NULL, _x("property"), _x(value_orig) );
				xmlNewProp( p, _x("name"), _x(name) );
			}
		}
	}
}

static void serialise_store_properties( serialise_context context, mlt_properties properties, xmlNode *node, const char *store )
{
	int i;
	xmlNode *p;

	// Enumerate the properties
	for ( i = 0; store != NULL && i < mlt_properties_count( properties ); i++ )
	{
		char *name = mlt_properties_get_name( properties, i );
		if ( !strncmp( name, store, strlen( store ) ) )
		{
			// Force a string with keyframes to be parsed with mlt_animation to
			// facilitate output with the specified time format.
			mlt_properties_anim_get_int( properties, name, 0, 0 );

			char *value = mlt_properties_get_value_tf( properties, i, context->time_format );
			if ( value )
			{
				int rootlen = strlen( context->root );
				// convert absolute path to relative
				if ( rootlen && !strncmp( value, context->root, rootlen ) && value[ rootlen ] == '/' )
					p = xmlNewTextChild( node, NULL, _x("property"), _x(value + rootlen + 1) );
				else
					p = xmlNewTextChild( node, NULL, _x("property"), _x(value) );
				xmlNewProp( p, _x("name"), _x(name) );
			}
		}
	}
}
<commit_msg>Fix serializing non-animation strings with '='.<commit_after>/*
 * consumer_xml.c -- a libxml2 serialiser of mlt service networks
 * Copyright (C) 2003-2018 Meltytech, LLC
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "common.h"

#include <framework/mlt.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <locale.h>
#include <libxml/tree.h>
#include <pthread.h>

#define ID_SIZE 128
#define TIME_PROPERTY "_consumer_xml"

#define _x (const xmlChar*)
#define _s (const char*)

// This maintains counters for adding ids to elements
struct serialise_context_s
{
	mlt_properties id_map;
	int producer_count;
	int multitrack_count;
	int playlist_count;
	int tractor_count;
	int filter_count;
	int transition_count;
	int pass;
	mlt_properties hide_map;
	char *root;
	char *store;
	int no_meta;
	mlt_profile profile;
	mlt_time_format time_format;
};
typedef struct serialise_context_s* serialise_context;

/** Forward references to static functions.
*/

static int consumer_start( mlt_consumer parent );
static int consumer_stop( mlt_consumer parent );
static int consumer_is_stopped( mlt_consumer consumer );
static void *consumer_thread( void *arg );
static void serialise_service( serialise_context context, mlt_service service, xmlNode *node );

typedef enum
{
	xml_existing,
	xml_producer,
	xml_multitrack,
	xml_playlist,
	xml_tractor,
	xml_filter,
	xml_transition
}
xml_type;

/** Create or retrieve an id associated to this service.
*/

static char *xml_get_id( serialise_context context, mlt_service service, xml_type type )
{
	char *id = NULL;
	int i = 0;
	mlt_properties map = context->id_map;

	// Search the map for the service
	for ( i = 0; i < mlt_properties_count( map ); i ++ )
		if ( mlt_properties_get_data_at( map, i, NULL ) == service )
			break;

	// If the service is not in the map, and the type indicates a new id is needed...
	if ( i >= mlt_properties_count( map ) && type != xml_existing )
	{
		// Attempt to reuse existing id
		id = mlt_properties_get( MLT_SERVICE_PROPERTIES( service ), "id" );

		// If no id, or the id is used in the map (for another service), then
		// create a new one.
		if ( id == NULL || mlt_properties_get_data( map, id, NULL ) != NULL )
		{
			char temp[ ID_SIZE ];
			do
			{
				switch( type )
				{
					case xml_producer:
						sprintf( temp, "producer%d", context->producer_count ++ );
						break;
					case xml_multitrack:
						sprintf( temp, "multitrack%d", context->multitrack_count ++ );
						break;
					case xml_playlist:
						sprintf( temp, "playlist%d", context->playlist_count ++ );
						break;
					case xml_tractor:
						sprintf( temp, "tractor%d", context->tractor_count ++ );
						break;
					case xml_filter:
						sprintf( temp, "filter%d", context->filter_count ++ );
						break;
					case xml_transition:
						sprintf( temp, "transition%d", context->transition_count ++ );
						break;
					case xml_existing:
						// Never gets here
						break;
				}
			}
			while( mlt_properties_get_data( map, temp, NULL ) != NULL );

			// Set the data at the generated name
			mlt_properties_set_data( map, temp, service, 0, NULL, NULL );

			// Get the pointer to the name (i is the end of the list)
			id = mlt_properties_get_name( map, i );
		}
		else
		{
			// Store the existing id in the map
			mlt_properties_set_data( map, id, service, 0, NULL, NULL );
		}
	}
	else if ( type == xml_existing )
	{
		id = mlt_properties_get_name( map, i );
	}

	return id;
}

/** This is what will be called by the factory - anything can be passed in
	via the argument, but keep it simple.
*/

mlt_consumer consumer_xml_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
{
	// Create the consumer object
	mlt_consumer consumer = calloc( 1, sizeof( struct mlt_consumer_s ) );

	// If no malloc'd and consumer init ok
	if ( consumer != NULL && mlt_consumer_init( consumer, NULL, profile ) == 0 )
	{
		// Allow thread to be started/stopped
		consumer->start = consumer_start;
		consumer->stop = consumer_stop;
		consumer->is_stopped = consumer_is_stopped;

		mlt_properties_set( MLT_CONSUMER_PROPERTIES( consumer ), "resource", arg );
		mlt_properties_set_int( MLT_CONSUMER_PROPERTIES( consumer ), "real_time", 0 );
		mlt_properties_set_int( MLT_CONSUMER_PROPERTIES( consumer ), "prefill", 1 );
		mlt_properties_set_int( MLT_CONSUMER_PROPERTIES( consumer ), "terminate_on_pause", 1 );

		// Return the consumer produced
		return consumer;
	}

	// malloc or consumer init failed
	free( consumer );

	// Indicate failure
	return NULL;
}

static void serialise_properties( serialise_context context, mlt_properties properties, xmlNode *node )
{
	int i;
	xmlNode *p;

	// Enumerate the properties
	for ( i = 0; i < mlt_properties_count( properties ); i++ )
	{
		char *name = mlt_properties_get_name( properties, i );
		if ( name != NULL &&
			 name[ 0 ] != '_' &&
			 mlt_properties_get_value( properties, i ) != NULL &&
			 ( !context->no_meta || strncmp( name, "meta.", 5 ) ) &&
			 strcmp( name, "mlt" ) &&
			 strcmp( name, "mlt_type" ) &&
			 strcmp( name, "in" ) &&
			 strcmp( name, "out" ) &&
			 strcmp( name, "id" ) &&
			 strcmp( name, "title" ) &&
			 strcmp( name, "root" ) &&
			 strcmp( name, "width" ) &&
			 strcmp( name, "height" ) )
		{
			char *value = mlt_properties_get_value_tf( properties, i, context->time_format );
			if ( value )
			{
				int rootlen = strlen( context->root );
				const char *value_orig = value;
				size_t prefix_size = mlt_xml_prefix_size( properties, name, value );

				// Strip off prefix.
				if ( prefix_size )
					value += prefix_size;

				// Ignore trailing slash on root.
				if ( rootlen && ( context->root[rootlen - 1] == '/' || context->root[rootlen - 1] == '\\') )
					--rootlen;

				// convert absolute path to relative
				if ( rootlen && !strncmp( value, context->root, rootlen ) &&
					( value[rootlen] == '/' || value[rootlen] == '\\' ) )
				{
					if ( prefix_size )
					{
						char *s = calloc( 1, strlen( value_orig ) - rootlen + 1 );
						strncat( s, value_orig, prefix_size );
						strcat( s, value + rootlen + 1 );
						p = xmlNewTextChild( node, NULL, _x("property"), _x(s) );
						free( s );
					} else {
						p = xmlNewTextChild( node, NULL, _x("property"), _x(value_orig + rootlen + 1) );
					}
				}
				else
					p = xmlNewTextChild( node, NULL, _x("property"), _x(value_orig) );
				xmlNewProp( p, _x("name"), _x(name) );
			}
		}
	}
}

static void serialise_store_properties( serialise_context context, mlt_properties properties, xmlNode *node, const char *store )
{
	int i;
	xmlNode *p;

	// Enumerate the properties
	for ( i = 0; store != NULL && i < mlt_properties_count( properties ); i++ )
	{
		char *name = mlt_properties_get_name( properties, i );
		if ( !strncmp( name, store, strlen( store ) ) )
		{
			char *value = mlt_properties_get_value_tf( properties, i, context->time_format );
			if ( value )
			{
				int rootlen = strlen( context->root );
				// convert absolute path to relative
				if ( rootlen && !strncmp( value, context->root, rootlen ) && value[ rootlen ] == '/' )
					p = xmlNewTextChild( node, NULL, _x("property"), _x(value + rootlen + 1) );
				else
					p = xmlNewTextChild( node, NULL, _x("property"), _x(value) );
				xmlNewProp( p, _x("name"), _x(name) );
			}
		}
	}
}

{'fraction_non_alphanumeric': 0.11803098694644382, 'fraction_numerical': 0.005733805050628278, 'mean_word_length': 3.404889844169801, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 76}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '100766', 'n_tokens_mistral': 6234, 'n_tokens_neox': 5381, 'n_words': 1711}