<commit_before>/*
 *			GPAC - Multimedia Framework C SDK
 *
 *			Authors: <NAME>
 *			Copyright (c) 2005-200X ENST
 *			All rights reserved
 *
 *  This file is part of GPAC / common tools sub-project
 *
 *  GPAC is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *   
 *  GPAC is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *   
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 
 *
 */


#include <gpac/xml.h>
#include <gpac/utf.h>
/*since 0.2.2, we use zlib for xmt/x3d reading to handle gz files*/
#include <zlib.h>

#if (defined(WIN32) || defined(_WIN32_WCE)) && !defined(__GNUC__)
#pragma comment(lib, "zlib")
#endif


static GF_Err gf_xml_sax_parse_intern(GF_SAXParser *parser, char *current);

static char *xml_translate_xml_string(char *str)
{
	char *value;
	u32 size, i, j;
	if (!str || !strlen(str)) return NULL;
	value = (char *)gf_malloc(sizeof(char) * 500);
	size = 500;
	i = j = 0;
	while (str[i]) {
		if (j+20 >= size) {
			size += 500;
			value = (char *)gf_realloc(value, sizeof(char)*size);
		}
		if (str[i] == '&') {
			if (str[i+1]=='#') {
				char szChar[20], *end;
				u16 wchar[2];
				u32 val;
				const unsigned short *srcp;
				strncpy(szChar, str+i, 10);
				end = strchr(szChar, ';');
				if (!end) break;
				end[1] = 0;
				i+=strlen(szChar);
				wchar[1] = 0;
				if (szChar[2]=='x')
					sscanf(szChar, "&#x%x;", &val);
				else
					sscanf(szChar, "&#%d;", &val);
				wchar[0] = val;
				srcp = wchar;
				j += gf_utf8_wcstombs(&value[j], 20, &srcp);
			}
			else if (!strnicmp(&str[i], "&amp;", sizeof(char)*5)) {
				value[j] = '&';
				j++;
				i+= 5;
			}
			else if (!strnicmp(&str[i], "&lt;", sizeof(char)*4)) {
				value[j] = '<';
				j++;
				i+= 4;
			}
			else if (!strnicmp(&str[i], "&gt;", sizeof(char)*4)) {
				value[j] = '>';
				j++;
				i+= 4;
			}
			else if (!strnicmp(&str[i], "&apos;", sizeof(char)*6)) {
				value[j] = '\'';
				j++;
				i+= 6;
			}
			else if (!strnicmp(&str[i], "&quot;", sizeof(char)*6)) {
				value[j] = '\"';
				j++;
				i+= 6;
			} else {
				value[j] = str[i];
				j++; i++;
			}
		} else {
			value[j] = str[i];
			j++; i++;
		}
	}
	value[j] = 0;
	return value;
}


enum
{
	SAX_STATE_ATT_NAME,
	SAX_STATE_ATT_VALUE,
	SAX_STATE_ELEMENT,
	SAX_STATE_COMMENT,
	SAX_STATE_TEXT_CONTENT,
	SAX_STATE_ENTITY,
	SAX_STATE_SKIP_DOCTYPE,
	SAX_STATE_CDATA,
	SAX_STATE_DONE,
	SAX_STATE_XML_PROC,
	SAX_STATE_SYNTAX_ERROR,
};

typedef struct
{
	u32 name_start, name_end;
	u32 val_start, val_end;
	Bool has_entities;
} GF_XMLSaxAttribute;


//#define NO_GZIP


struct _tag_sax_parser
{
	/*0: UTF-8, 1: UTF-16 BE, 2: UTF-16 LE. String input is always converted back to utf8*/
	s32 unicode_type;
	char *buffer;
	/*alloc size, line size and current position*/
	u32 alloc_size, line_size, current_pos;
	/*current node depth*/
	u32 node_depth;

	/*gz input file*/
#ifdef NO_GZIP
	FILE *f_in;
#else
	gzFile gz_in;
#endif
	/*current line , file size and pos for user notif*/
	u32 line, file_size, file_pos;

	/*SAX callbacks*/
	gf_xml_sax_node_start sax_node_start;
	gf_xml_sax_node_end sax_node_end;
	gf_xml_sax_text_content sax_text_content;
	void *sax_cbck;
	gf_xml_sax_progress on_progress;

	u32 sax_state;
	u32 init_state;
	GF_List *entities;
	char att_sep;
	Bool in_entity, suspended;
	u32 in_quote;

	u32 elt_start_pos, elt_end_pos;

	/*last error found*/
	char err_msg[1000];

	u32 att_name_start, elt_name_start, elt_name_end, text_start, text_end;

	GF_XMLAttribute *attrs;
	GF_XMLSaxAttribute *sax_attrs;
	u32 nb_attrs, nb_alloc_attrs;
};

static GF_XMLSaxAttribute *xml_get_sax_attribute(GF_SAXParser *parser)
{
	if (parser->nb_attrs==parser->nb_alloc_attrs) {
		parser->nb_alloc_attrs++;
		parser->sax_attrs = (GF_XMLSaxAttribute *)gf_realloc(parser->sax_attrs, sizeof(GF_XMLSaxAttribute)*parser->nb_alloc_attrs);
		parser->attrs = (GF_XMLAttribute *)gf_realloc(parser->attrs, sizeof(GF_XMLAttribute)*parser->nb_alloc_attrs);
	}
	return &parser->sax_attrs[parser->nb_attrs++];
}

static void xml_sax_swap(GF_SAXParser *parser)
{
	if (parser->current_pos && ((parser->sax_state==SAX_STATE_TEXT_CONTENT) || (parser->sax_state==SAX_STATE_COMMENT) ) ) {
		assert(parser->line_size >= parser->current_pos);
		parser->line_size -= parser->current_pos;
		parser->file_pos += parser->current_pos;
		if (parser->line_size) memmove(parser->buffer, parser->buffer + parser->current_pos, sizeof(char)*parser->line_size); 
		parser->buffer[parser->line_size] = 0;
		parser->current_pos = 0;
	}
}

static void format_sax_error(GF_SAXParser *parser, u32 linepos, const char* fmt, ...)
{
	va_list args;
	u32 len;
	char szM[20];

	va_start(args, fmt);
	vsprintf(parser->err_msg, fmt, args);
	va_end(args);
	
	sprintf(szM, " - Line %d: ", parser->line + 1);
	strcat(parser->err_msg, szM);
	len = strlen(parser->err_msg);
	strncpy(parser->err_msg + len, parser->buffer+ (linepos ? linepos : parser->current_pos), 10);
	parser->err_msg[len + 10] = 0;
	parser->sax_state = SAX_STATE_SYNTAX_ERROR;
}

static void xml_sax_node_end(GF_SAXParser *parser, Bool had_children)
{
	char *name, *sep, c;

	assert(parser->elt_name_start);
	assert(parser->elt_name_end);
	if (!parser->node_depth) {
		format_sax_error(parser, 0, "Markup error");
		return;
	}
	c = parser->buffer[parser->elt_name_end - 1];
	parser->buffer[parser->elt_name_end - 1] = 0;
	name = parser->buffer + parser->elt_name_start - 1;
	
	if (parser->sax_node_end) {
		sep = strchr(name, ':');
		if (sep) {
			sep[0] = 0;
			parser->sax_node_end(parser->sax_cbck, sep+1, name);
			sep[0] = ':';
		} else {
			parser->sax_node_end(parser->sax_cbck, name, NULL);
		}
	}
	parser->buffer[parser->elt_name_end - 1] = c;
	parser->node_depth--;
	if (!parser->init_state && !parser->node_depth) parser->sax_state = SAX_STATE_DONE;
	xml_sax_swap(parser);
	parser->text_start = parser->text_end = 0;
}

static void xml_sax_node_start(GF_SAXParser *parser)
{
	Bool has_entities = 0;
	u32 i;
	char *sep, c, *name;

	assert(parser->elt_name_start && parser->elt_name_end);
	c = parser->buffer[parser->elt_name_end - 1];
	parser->buffer[parser->elt_name_end - 1] = 0;
	name = parser->buffer + parser->elt_name_start - 1;

	for (i=0;i<parser->nb_attrs; i++) {
		parser->attrs[i].name = parser->buffer + parser->sax_attrs[i].name_start - 1;
		parser->buffer[parser->sax_attrs[i].name_end-1] = 0;
		parser->attrs[i].value = parser->buffer + parser->sax_attrs[i].val_start - 1;
		parser->buffer[parser->sax_attrs[i].val_end-1] = 0;

		if (strchr(parser->attrs[i].value, '&')) {
			parser->sax_attrs[i].has_entities = 1;
			has_entities = 1;
			parser->attrs[i].value = xml_translate_xml_string(parser->attrs[i].value);
		}
		/*store first char pos after current attrib for node peeking*/
		parser->att_name_start = parser->sax_attrs[i].val_end;
	}

	if (parser->sax_node_start) {
		sep = strchr(name, ':');
		if (sep) {
			sep[0] = 0;
			parser->sax_node_start(parser->sax_cbck, sep+1, name, parser->attrs, parser->nb_attrs);
			sep[0] = ':';
		} else {
			parser->sax_node_start(parser->sax_cbck, name, NULL, parser->attrs, parser->nb_attrs);
		}
	}
	parser->att_name_start = 0;
	parser->buffer[parser->elt_name_end - 1] = c;
	parser->node_depth++;
	if (has_entities) {
		for (i=0;i<parser->nb_attrs; i++) {
			if (parser->sax_attrs[i].has_entities) {
				parser->sax_attrs[i].has_entities = 0;
				gf_free(parser->attrs[i].value);
			}
		}
	}
	parser->nb_attrs = 0;
	xml_sax_swap(parser);
	parser->text_start = parser->text_end = 0;
}

static Bool xml_sax_parse_attribute(GF_SAXParser *parser)
{
	char *sep;
	GF_XMLSaxAttribute *att = NULL;

	/*looking for attribute name*/
	if (parser->sax_state==SAX_STATE_ATT_NAME) {
		/*looking for start*/
		if (!parser->att_name_start) {
			while (parser->current_pos < parser->line_size) {
				u8 c = parser->buffer[parser->current_pos];
				switch (c) {
				case '\n':
					parser->line++;
				case ' ':
				case '\r':
				case '\t':
					parser->current_pos++;
					continue;
				/*end of element*/
				case '?':
					if (parser->init_state!=1) break;
				case '/':
					/*not enough data*/
					if (parser->current_pos+1 == parser->line_size) return 1;
					if (parser->buffer[parser->current_pos+1]=='>') {
						parser->current_pos+=2;
						parser->elt_end_pos = parser->file_pos + parser->current_pos - 1;
						/*done parsing attr AND elements*/
						if (!parser->init_state) {
							xml_sax_node_start(parser);
							/*move to SAX_STATE_TEXT_CONTENT to force text flush*/
							parser->sax_state = SAX_STATE_TEXT_CONTENT;
							xml_sax_node_end(parser, 0);
						} else {
							parser->nb_attrs = 0;
						}
						parser->sax_state = (parser->init_state) ? SAX_STATE_ELEMENT : SAX_STATE_TEXT_CONTENT;
						parser->text_start = parser->text_end = 0;
						return 0;
					}
					if (!parser->in_quote && (c=='/')) {
						if (!parser->init_state) {
							format_sax_error(parser, 0, "Markup error");
							return 1;
						}
					}
					break;
				case '"':
					if (parser->sax_state==SAX_STATE_ATT_VALUE) break;
					if (parser->in_quote && (parser->in_quote!=c) ) {
						format_sax_error(parser, 0, "Markup error");
						return 1;
					}
					if (parser->in_quote) parser->in_quote = 0;
					else parser->in_quote = c;
					break;
				case '>':
					parser->current_pos+=1;
					/*end of <!DOCTYPE>*/
					if (parser->init_state) {
						if (parser->init_state==1) {
							format_sax_error(parser, 0, "Invalid DOCTYPE");
							return 1;
						}
						parser->sax_state = SAX_STATE_ELEMENT;
						return 0;
					}
					/*done parsing attr*/
					parser->sax_state = SAX_STATE_TEXT_CONTENT;
					xml_sax_node_start(parser);
					return 0;
				case '[':
					if (parser->init_state) {
						parser->current_pos+=1;
						if (parser->init_state==1) {
							format_sax_error(parser, 0, "Invalid DOCTYPE");
							return 1;
						}
						parser->sax_state = SAX_STATE_ELEMENT;
						return 0;
					}
					break;
				case '<':
					format_sax_error(parser, 0, "Invalid character '<'");
					return 0;
				/*first char of attr name*/
				default:
					parser->att_name_start = parser->current_pos + 1;
					break;
				}
				parser->current_pos++;
				if (parser->att_name_start) break;
			}
			if (parser->current_pos == parser->line_size) return 1;
		}

		if (parser->init_state==2) {
			sep = strchr(parser->buffer + parser->att_name_start - 1, parser->in_quote ?  parser->in_quote : ' ');
			/*not enough data*/
			if (!sep) return 1;
			parser->current_pos = sep - parser->buffer;
			parser->att_name_start = 0;
			if (parser->in_quote) {
				parser->current_pos++;
				parser->in_quote = 0;
			}
			return 0;
		}

		/*looking for '"'*/
		if (parser->att_name_start) {
			u32 i, first=1;
			sep = strchr(parser->buffer + parser->att_name_start - 1, '=');
			/*not enough data*/
			if (!sep) return 1;

			parser->current_pos = sep - parser->buffer;
			att = xml_get_sax_attribute(parser);
			att->name_start = parser->att_name_start;
			att->name_end = parser->current_pos + 1;
			while (strchr(" \n\t", parser->buffer[att->name_end - 2])) {
				assert(att->name_end);
				att->name_end --;
			}
			att->has_entities = 0;

			for (i=att->name_start; i<att->name_end; i++) {
				char c = parser->buffer[i-1];
				if ((c>='a') && (c<='z')) {}
				else if ((c>='A') && (c<='Z')) {}
				else if ((c==':') || (c=='_')) {}

				else if (!first && ((c=='-') || (c=='.') || ((c>='0') && (c<='9')) )) {}

				else {
					format_sax_error(parser, att->name_start-1, "Invalid character \'%c\' for attribute name", c);
					return 1;
				}

				first=0;
			}

			parser->att_name_start = 0;
			parser->current_pos++;
			parser->sax_state = SAX_STATE_ATT_VALUE;

		}
	}
	
	if (parser->sax_state == SAX_STATE_ATT_VALUE) {
		att = &parser->sax_attrs[parser->nb_attrs-1];
		/*looking for first delimiter*/
		if (!parser->att_sep) {
			while (parser->current_pos < parser->line_size) {
				u8 c = parser->buffer[parser->current_pos];
				switch (c) {
				case '\n':
					parser->line++;
				case ' ':
				case '\r':
				case '\t':
					parser->current_pos++;
					continue;
				case '\'':
				case '"':
					parser->att_sep = c;
					att->val_start = parser->current_pos + 2;
					break;
				default:
					break;
				}
				parser->current_pos++;
				if (parser->att_sep) break;
			}
			if (parser->current_pos == parser->line_size) return 1;
		} 

att_retry:

		assert(parser->att_sep);
		sep = strchr(parser->buffer + parser->current_pos, parser->att_sep);
		if (!sep || !sep[1]) return 1;

		if (!parser->init_state && (strchr(" />\n\t\r", sep[1])==NULL)) {
			parser->current_pos = sep - parser->buffer + 1;
			goto att_retry;
		}

		parser->current_pos = sep - parser->buffer;
		att->val_end = parser->current_pos + 1;
		parser->current_pos++;

		/*"style" always at the begining of the attributes for ease of parsing*/
		if (!strncmp(parser->buffer + att->name_start-1, "style", 5)) {
			GF_XMLSaxAttribute prev = parser->sax_attrs[0];
			parser->sax_attrs[0] = *att;
			*att = prev;
		}
		parser->att_sep = 0;
		parser->sax_state = SAX_STATE_ATT_NAME;
		parser->att_name_start = 0;
		return 0;
	}
	return 1;
}


typedef struct
{
	char *name;
	char *value;
	u32 namelen;
	u8 sep;
} XML_Entity;

static void xml_sax_flush_text(GF_SAXParser *parser)
{
	char *text, c;
	if (!parser->text_start || parser->init_state || !parser->sax_text_content) return;

	assert(parser->text_start < parser->text_end);

	c = parser->buffer[parser->text_end-1];
	parser->buffer[parser->text_end-1] = 0;
	text = parser->buffer + parser->text_start-1;

	/*solve XML built-in entities*/
	if (strchr(text, '&') && strchr(text, ';')) {
		char *xml_text = xml_translate_xml_string(text);
		if (xml_text) {
			parser->sax_text_content(parser->sax_cbck, xml_text, (parser->sax_state==SAX_STATE_CDATA) ? 1 : 0);
			gf_free(xml_text);
		}
	} else {
		parser->sax_text_content(parser->sax_cbck, text, (parser->sax_state==SAX_STATE_CDATA) ? 1 : 0);
	}
	parser->buffer[parser->text_end-1] = c;
	parser->text_start = parser->text_end = 0;
}

static void xml_sax_store_text(GF_SAXParser *parser, u32 txt_len)
{
	if (!txt_len) return;

	if (!parser->text_start) {
		parser->text_start = parser->current_pos + 1;
		parser->text_end = parser->text_start + txt_len;
		parser->current_pos += txt_len;
		assert(parser->current_pos <= parser->line_size);
		return;
	}
	/*contiguous text*/
	if (parser->text_end && (parser->text_end-1 == parser->current_pos)) {
		parser->text_end += txt_len;
		parser->current_pos += txt_len;
		assert(parser->current_pos <= parser->line_size);
		return;
	}
	/*need to flush*/
	xml_sax_flush_text(parser);

	parser->text_start = parser->current_pos + 1;
	parser->text_end = parser->text_start + txt_len;
	parser->current_pos += txt_len;
	assert(parser->current_pos <= parser->line_size);
}

static char *xml_get_current_text(GF_SAXParser *parser)
{
	char *text, c;
	if (!parser->text_start) return NULL;

	c = parser->buffer[parser->text_end-1];
	parser->buffer[parser->text_end-1] = 0;
	text = gf_strdup(parser->buffer + parser->text_start-1);
	parser->buffer[parser->text_end-1] = c;
	parser->text_start = parser->text_end = 0;
	return text;
}

static void xml_sax_skip_doctype(GF_SAXParser *parser)
{
	while (parser->current_pos < parser->line_size) {
		if (parser->buffer[parser->current_pos]=='>') {
			parser->sax_state = SAX_STATE_ELEMENT;
			parser->current_pos++;
			xml_sax_swap(parser);
			return;
		}
		parser->current_pos++;
	}
}

static void xml_sax_skip_xml_proc(GF_SAXParser *parser)
{
	while (parser->current_pos + 1 < parser->line_size) {
		if ((parser->buffer[parser->current_pos]=='?') && (parser->buffer[parser->current_pos+1]=='>')) {
			parser->sax_state = SAX_STATE_ELEMENT;
			parser->current_pos++;
			xml_sax_swap(parser);
			return;
		}
		parser->current_pos++;
	}
}


static void xml_sax_parse_entity(GF_SAXParser *parser)
{
	char szName[1024];
	u32 i = 0;
	XML_Entity *ent = (XML_Entity *)gf_list_last(parser->entities);
	char *skip_chars = " \t\n\r";
	i=0;
	if (ent && ent->value) ent = NULL;
	if (ent) skip_chars = NULL;

	while (parser->current_pos+i < parser->line_size) {
		u8 c = parser->buffer[parser->current_pos+i];
		if (skip_chars && strchr(skip_chars, c)) {
			if (c=='\n') parser->line++;
			parser->current_pos++;
			continue;
		}
		if (!ent && (c=='%')) {
			parser->current_pos+=i+1;
			parser->sax_state = SAX_STATE_SKIP_DOCTYPE;
			return;
		}
		else if (!ent && ((c=='\"') || (c=='\'')) ) {
			szName[i] = 0;
			GF_SAFEALLOC(ent, XML_Entity);
			ent->name = gf_strdup(szName);
			ent->namelen = strlen(ent->name);
			ent->sep = c;
			parser->current_pos += 1+i;
			assert(parser->current_pos < parser->line_size);
			xml_sax_swap(parser);
			i=0;
			gf_list_add(parser->entities, ent);
			skip_chars = NULL;
		} else if (ent && c==ent->sep) {
			xml_sax_store_text(parser, i);

			ent->value = xml_get_current_text(parser);
			if (!ent->value) ent->value = gf_strdup("");

			parser->current_pos += 1;
			assert(parser->current_pos < parser->line_size);
			xml_sax_swap(parser);
			parser->sax_state = SAX_STATE_SKIP_DOCTYPE;
			return;
		} else if (!ent) {
			szName[i] = c;
			i++;
		} else {
			i++;
		}
	}
	xml_sax_store_text(parser, i);
}

static void xml_sax_cdata(GF_SAXParser *parser)
{
	char *cd_end = strstr(parser->buffer + parser->current_pos, "]]>");
	if (!cd_end) {
		xml_sax_store_text(parser, parser->line_size - parser->current_pos);
	} else {
		u32 size = cd_end - (parser->buffer + parser->current_pos);
		xml_sax_store_text(parser, size);
		xml_sax_flush_text(parser);
		parser->current_pos += 3;
		assert(parser->current_pos <= parser->line_size);
		parser->sax_state = SAX_STATE_TEXT_CONTENT;
	}
}

static Bool xml_sax_parse_comments(GF_SAXParser *parser)
{
	char *end = strstr(parser->buffer + parser->current_pos, "-->");
	if (!end) {
		if (parser->line_size>3)
			parser->current_pos = parser->line_size-3;
		xml_sax_swap(parser);
		return 0;
	}

	parser->current_pos += 3 + (u32) (end - (parser->buffer + parser->current_pos) );
	assert(parser->current_pos <= parser->line_size);
	parser->sax_state = SAX_STATE_TEXT_CONTENT;
	parser->text_start = parser->text_end = 0;
	xml_sax_swap(parser);
	return 1;
}



static GF_Err xml_sax_parse(GF_SAXParser *parser, Bool force_parse)
{
	u32 i = 0;
	Bool is_text, is_end;
	u8 c;
	char *elt, sep;
	u32 cdata_sep;

	is_text = 0;
	while (parser->current_pos<parser->line_size) {
		if (!force_parse && parser->suspended) goto exit;

restart:
		is_text = 0;
		switch (parser->sax_state) {
		/*load an XML element*/
		case SAX_STATE_TEXT_CONTENT:
			is_text = 1;
		case SAX_STATE_ELEMENT:
			elt = NULL;
			i=0;
			while ((c = parser->buffer[parser->current_pos+i]) !='<') {
				if ((parser->init_state==2) && (c ==']')) {
					parser->sax_state = SAX_STATE_ATT_NAME;
					parser->current_pos+=i+1;
					goto restart;
				}
				i++;
				if (c=='\n') parser->line++;
				if (parser->current_pos+i==parser->line_size) goto exit;
			}
			if (is_text && i) {
				xml_sax_store_text(parser, i);
				is_text = 0;
				parser->sax_state = SAX_STATE_ELEMENT;
			} else if (i) {
				parser->current_pos += i;
				assert(parser->current_pos < parser->line_size);
			}
			is_end = 0;
			i = 0;
			cdata_sep = 0;
			while (1) {
				char c = parser->buffer[parser->current_pos+1+i];
				if (!c) {
					i = 0;
					goto exit;
				}
				if ((c=='\t') || (c=='\r') || (c==' ') ) {
					if (i) break;
					else parser->current_pos++;
				}
				else if (c=='\n') {
					parser->line++;
					if (i) break;
					else parser->current_pos++;
				}
				else if (c=='>') break;
				else if (c=='=') break;
				else if (c=='[') {
					i++;
					if (!cdata_sep) cdata_sep = 1;
					else {
						break;
					}
				}
				else if (c=='/') {
					is_end = !i ? 1 : 2;
					i++;
				} else if (c=='<') {
					parser->sax_state = SAX_STATE_SYNTAX_ERROR;
					return GF_CORRUPTED_DATA;
				} else {
					i++;
				}<commit_msg>Fix for parsing comments in xml files, especially when comments are as follows:   <!--<g id="draft-watermark"> The < is present right after the !-- and was mistakenly interpreted Maybe a general fix should be needed here<commit_after>/*
 *			GPAC - Multimedia Framework C SDK
 *
 *			Authors: <NAME>
 *			Copyright (c) 2005-200X ENST
 *			All rights reserved
 *
 *  This file is part of GPAC / common tools sub-project
 *
 *  GPAC is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU Lesser General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *   
 *  GPAC is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *   
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 
 *
 */


#include <gpac/xml.h>
#include <gpac/utf.h>
/*since 0.2.2, we use zlib for xmt/x3d reading to handle gz files*/
#include <zlib.h>

#if (defined(WIN32) || defined(_WIN32_WCE)) && !defined(__GNUC__)
#pragma comment(lib, "zlib")
#endif


static GF_Err gf_xml_sax_parse_intern(GF_SAXParser *parser, char *current);

static char *xml_translate_xml_string(char *str)
{
	char *value;
	u32 size, i, j;
	if (!str || !strlen(str)) return NULL;
	value = (char *)gf_malloc(sizeof(char) * 500);
	size = 500;
	i = j = 0;
	while (str[i]) {
		if (j+20 >= size) {
			size += 500;
			value = (char *)gf_realloc(value, sizeof(char)*size);
		}
		if (str[i] == '&') {
			if (str[i+1]=='#') {
				char szChar[20], *end;
				u16 wchar[2];
				u32 val;
				const unsigned short *srcp;
				strncpy(szChar, str+i, 10);
				end = strchr(szChar, ';');
				if (!end) break;
				end[1] = 0;
				i+=strlen(szChar);
				wchar[1] = 0;
				if (szChar[2]=='x')
					sscanf(szChar, "&#x%x;", &val);
				else
					sscanf(szChar, "&#%d;", &val);
				wchar[0] = val;
				srcp = wchar;
				j += gf_utf8_wcstombs(&value[j], 20, &srcp);
			}
			else if (!strnicmp(&str[i], "&amp;", sizeof(char)*5)) {
				value[j] = '&';
				j++;
				i+= 5;
			}
			else if (!strnicmp(&str[i], "&lt;", sizeof(char)*4)) {
				value[j] = '<';
				j++;
				i+= 4;
			}
			else if (!strnicmp(&str[i], "&gt;", sizeof(char)*4)) {
				value[j] = '>';
				j++;
				i+= 4;
			}
			else if (!strnicmp(&str[i], "&apos;", sizeof(char)*6)) {
				value[j] = '\'';
				j++;
				i+= 6;
			}
			else if (!strnicmp(&str[i], "&quot;", sizeof(char)*6)) {
				value[j] = '\"';
				j++;
				i+= 6;
			} else {
				value[j] = str[i];
				j++; i++;
			}
		} else {
			value[j] = str[i];
			j++; i++;
		}
	}
	value[j] = 0;
	return value;
}


enum
{
	SAX_STATE_ATT_NAME,
	SAX_STATE_ATT_VALUE,
	SAX_STATE_ELEMENT,
	SAX_STATE_COMMENT,
	SAX_STATE_TEXT_CONTENT,
	SAX_STATE_ENTITY,
	SAX_STATE_SKIP_DOCTYPE,
	SAX_STATE_CDATA,
	SAX_STATE_DONE,
	SAX_STATE_XML_PROC,
	SAX_STATE_SYNTAX_ERROR,
};

typedef struct
{
	u32 name_start, name_end;
	u32 val_start, val_end;
	Bool has_entities;
} GF_XMLSaxAttribute;


//#define NO_GZIP


struct _tag_sax_parser
{
	/*0: UTF-8, 1: UTF-16 BE, 2: UTF-16 LE. String input is always converted back to utf8*/
	s32 unicode_type;
	char *buffer;
	/*alloc size, line size and current position*/
	u32 alloc_size, line_size, current_pos;
	/*current node depth*/
	u32 node_depth;

	/*gz input file*/
#ifdef NO_GZIP
	FILE *f_in;
#else
	gzFile gz_in;
#endif
	/*current line , file size and pos for user notif*/
	u32 line, file_size, file_pos;

	/*SAX callbacks*/
	gf_xml_sax_node_start sax_node_start;
	gf_xml_sax_node_end sax_node_end;
	gf_xml_sax_text_content sax_text_content;
	void *sax_cbck;
	gf_xml_sax_progress on_progress;

	u32 sax_state;
	u32 init_state;
	GF_List *entities;
	char att_sep;
	Bool in_entity, suspended;
	u32 in_quote;

	u32 elt_start_pos, elt_end_pos;

	/*last error found*/
	char err_msg[1000];

	u32 att_name_start, elt_name_start, elt_name_end, text_start, text_end;

	GF_XMLAttribute *attrs;
	GF_XMLSaxAttribute *sax_attrs;
	u32 nb_attrs, nb_alloc_attrs;
};

static GF_XMLSaxAttribute *xml_get_sax_attribute(GF_SAXParser *parser)
{
	if (parser->nb_attrs==parser->nb_alloc_attrs) {
		parser->nb_alloc_attrs++;
		parser->sax_attrs = (GF_XMLSaxAttribute *)gf_realloc(parser->sax_attrs, sizeof(GF_XMLSaxAttribute)*parser->nb_alloc_attrs);
		parser->attrs = (GF_XMLAttribute *)gf_realloc(parser->attrs, sizeof(GF_XMLAttribute)*parser->nb_alloc_attrs);
	}
	return &parser->sax_attrs[parser->nb_attrs++];
}

static void xml_sax_swap(GF_SAXParser *parser)
{
	if (parser->current_pos && ((parser->sax_state==SAX_STATE_TEXT_CONTENT) || (parser->sax_state==SAX_STATE_COMMENT) ) ) {
		assert(parser->line_size >= parser->current_pos);
		parser->line_size -= parser->current_pos;
		parser->file_pos += parser->current_pos;
		if (parser->line_size) memmove(parser->buffer, parser->buffer + parser->current_pos, sizeof(char)*parser->line_size); 
		parser->buffer[parser->line_size] = 0;
		parser->current_pos = 0;
	}
}

static void format_sax_error(GF_SAXParser *parser, u32 linepos, const char* fmt, ...)
{
	va_list args;
	u32 len;
	char szM[20];

	va_start(args, fmt);
	vsprintf(parser->err_msg, fmt, args);
	va_end(args);
	
	sprintf(szM, " - Line %d: ", parser->line + 1);
	strcat(parser->err_msg, szM);
	len = strlen(parser->err_msg);
	strncpy(parser->err_msg + len, parser->buffer+ (linepos ? linepos : parser->current_pos), 10);
	parser->err_msg[len + 10] = 0;
	parser->sax_state = SAX_STATE_SYNTAX_ERROR;
}

static void xml_sax_node_end(GF_SAXParser *parser, Bool had_children)
{
	char *name, *sep, c;

	assert(parser->elt_name_start);
	assert(parser->elt_name_end);
	if (!parser->node_depth) {
		format_sax_error(parser, 0, "Markup error");
		return;
	}
	c = parser->buffer[parser->elt_name_end - 1];
	parser->buffer[parser->elt_name_end - 1] = 0;
	name = parser->buffer + parser->elt_name_start - 1;
	
	if (parser->sax_node_end) {
		sep = strchr(name, ':');
		if (sep) {
			sep[0] = 0;
			parser->sax_node_end(parser->sax_cbck, sep+1, name);
			sep[0] = ':';
		} else {
			parser->sax_node_end(parser->sax_cbck, name, NULL);
		}
	}
	parser->buffer[parser->elt_name_end - 1] = c;
	parser->node_depth--;
	if (!parser->init_state && !parser->node_depth) parser->sax_state = SAX_STATE_DONE;
	xml_sax_swap(parser);
	parser->text_start = parser->text_end = 0;
}

static void xml_sax_node_start(GF_SAXParser *parser)
{
	Bool has_entities = 0;
	u32 i;
	char *sep, c, *name;

	assert(parser->elt_name_start && parser->elt_name_end);
	c = parser->buffer[parser->elt_name_end - 1];
	parser->buffer[parser->elt_name_end - 1] = 0;
	name = parser->buffer + parser->elt_name_start - 1;

	for (i=0;i<parser->nb_attrs; i++) {
		parser->attrs[i].name = parser->buffer + parser->sax_attrs[i].name_start - 1;
		parser->buffer[parser->sax_attrs[i].name_end-1] = 0;
		parser->attrs[i].value = parser->buffer + parser->sax_attrs[i].val_start - 1;
		parser->buffer[parser->sax_attrs[i].val_end-1] = 0;

		if (strchr(parser->attrs[i].value, '&')) {
			parser->sax_attrs[i].has_entities = 1;
			has_entities = 1;
			parser->attrs[i].value = xml_translate_xml_string(parser->attrs[i].value);
		}
		/*store first char pos after current attrib for node peeking*/
		parser->att_name_start = parser->sax_attrs[i].val_end;
	}

	if (parser->sax_node_start) {
		sep = strchr(name, ':');
		if (sep) {
			sep[0] = 0;
			parser->sax_node_start(parser->sax_cbck, sep+1, name, parser->attrs, parser->nb_attrs);
			sep[0] = ':';
		} else {
			parser->sax_node_start(parser->sax_cbck, name, NULL, parser->attrs, parser->nb_attrs);
		}
	}
	parser->att_name_start = 0;
	parser->buffer[parser->elt_name_end - 1] = c;
	parser->node_depth++;
	if (has_entities) {
		for (i=0;i<parser->nb_attrs; i++) {
			if (parser->sax_attrs[i].has_entities) {
				parser->sax_attrs[i].has_entities = 0;
				gf_free(parser->attrs[i].value);
			}
		}
	}
	parser->nb_attrs = 0;
	xml_sax_swap(parser);
	parser->text_start = parser->text_end = 0;
}

static Bool xml_sax_parse_attribute(GF_SAXParser *parser)
{
	char *sep;
	GF_XMLSaxAttribute *att = NULL;

	/*looking for attribute name*/
	if (parser->sax_state==SAX_STATE_ATT_NAME) {
		/*looking for start*/
		if (!parser->att_name_start) {
			while (parser->current_pos < parser->line_size) {
				u8 c = parser->buffer[parser->current_pos];
				switch (c) {
				case '\n':
					parser->line++;
				case ' ':
				case '\r':
				case '\t':
					parser->current_pos++;
					continue;
				/*end of element*/
				case '?':
					if (parser->init_state!=1) break;
				case '/':
					/*not enough data*/
					if (parser->current_pos+1 == parser->line_size) return 1;
					if (parser->buffer[parser->current_pos+1]=='>') {
						parser->current_pos+=2;
						parser->elt_end_pos = parser->file_pos + parser->current_pos - 1;
						/*done parsing attr AND elements*/
						if (!parser->init_state) {
							xml_sax_node_start(parser);
							/*move to SAX_STATE_TEXT_CONTENT to force text flush*/
							parser->sax_state = SAX_STATE_TEXT_CONTENT;
							xml_sax_node_end(parser, 0);
						} else {
							parser->nb_attrs = 0;
						}
						parser->sax_state = (parser->init_state) ? SAX_STATE_ELEMENT : SAX_STATE_TEXT_CONTENT;
						parser->text_start = parser->text_end = 0;
						return 0;
					}
					if (!parser->in_quote && (c=='/')) {
						if (!parser->init_state) {
							format_sax_error(parser, 0, "Markup error");
							return 1;
						}
					}
					break;
				case '"':
					if (parser->sax_state==SAX_STATE_ATT_VALUE) break;
					if (parser->in_quote && (parser->in_quote!=c) ) {
						format_sax_error(parser, 0, "Markup error");
						return 1;
					}
					if (parser->in_quote) parser->in_quote = 0;
					else parser->in_quote = c;
					break;
				case '>':
					parser->current_pos+=1;
					/*end of <!DOCTYPE>*/
					if (parser->init_state) {
						if (parser->init_state==1) {
							format_sax_error(parser, 0, "Invalid DOCTYPE");
							return 1;
						}
						parser->sax_state = SAX_STATE_ELEMENT;
						return 0;
					}
					/*done parsing attr*/
					parser->sax_state = SAX_STATE_TEXT_CONTENT;
					xml_sax_node_start(parser);
					return 0;
				case '[':
					if (parser->init_state) {
						parser->current_pos+=1;
						if (parser->init_state==1) {
							format_sax_error(parser, 0, "Invalid DOCTYPE");
							return 1;
						}
						parser->sax_state = SAX_STATE_ELEMENT;
						return 0;
					}
					break;
				case '<':
					format_sax_error(parser, 0, "Invalid character '<'");
					return 0;
				/*first char of attr name*/
				default:
					parser->att_name_start = parser->current_pos + 1;
					break;
				}
				parser->current_pos++;
				if (parser->att_name_start) break;
			}
			if (parser->current_pos == parser->line_size) return 1;
		}

		if (parser->init_state==2) {
			sep = strchr(parser->buffer + parser->att_name_start - 1, parser->in_quote ?  parser->in_quote : ' ');
			/*not enough data*/
			if (!sep) return 1;
			parser->current_pos = sep - parser->buffer;
			parser->att_name_start = 0;
			if (parser->in_quote) {
				parser->current_pos++;
				parser->in_quote = 0;
			}
			return 0;
		}

		/*looking for '"'*/
		if (parser->att_name_start) {
			u32 i, first=1;
			sep = strchr(parser->buffer + parser->att_name_start - 1, '=');
			/*not enough data*/
			if (!sep) return 1;

			parser->current_pos = sep - parser->buffer;
			att = xml_get_sax_attribute(parser);
			att->name_start = parser->att_name_start;
			att->name_end = parser->current_pos + 1;
			while (strchr(" \n\t", parser->buffer[att->name_end - 2])) {
				assert(att->name_end);
				att->name_end --;
			}
			att->has_entities = 0;

			for (i=att->name_start; i<att->name_end; i++) {
				char c = parser->buffer[i-1];
				if ((c>='a') && (c<='z')) {}
				else if ((c>='A') && (c<='Z')) {}
				else if ((c==':') || (c=='_')) {}

				else if (!first && ((c=='-') || (c=='.') || ((c>='0') && (c<='9')) )) {}

				else {
					format_sax_error(parser, att->name_start-1, "Invalid character \'%c\' for attribute name", c);
					return 1;
				}

				first=0;
			}

			parser->att_name_start = 0;
			parser->current_pos++;
			parser->sax_state = SAX_STATE_ATT_VALUE;

		}
	}
	
	if (parser->sax_state == SAX_STATE_ATT_VALUE) {
		att = &parser->sax_attrs[parser->nb_attrs-1];
		/*looking for first delimiter*/
		if (!parser->att_sep) {
			while (parser->current_pos < parser->line_size) {
				u8 c = parser->buffer[parser->current_pos];
				switch (c) {
				case '\n':
					parser->line++;
				case ' ':
				case '\r':
				case '\t':
					parser->current_pos++;
					continue;
				case '\'':
				case '"':
					parser->att_sep = c;
					att->val_start = parser->current_pos + 2;
					break;
				default:
					break;
				}
				parser->current_pos++;
				if (parser->att_sep) break;
			}
			if (parser->current_pos == parser->line_size) return 1;
		} 

att_retry:

		assert(parser->att_sep);
		sep = strchr(parser->buffer + parser->current_pos, parser->att_sep);
		if (!sep || !sep[1]) return 1;

		if (!parser->init_state && (strchr(" />\n\t\r", sep[1])==NULL)) {
			parser->current_pos = sep - parser->buffer + 1;
			goto att_retry;
		}

		parser->current_pos = sep - parser->buffer;
		att->val_end = parser->current_pos + 1;
		parser->current_pos++;

		/*"style" always at the begining of the attributes for ease of parsing*/
		if (!strncmp(parser->buffer + att->name_start-1, "style", 5)) {
			GF_XMLSaxAttribute prev = parser->sax_attrs[0];
			parser->sax_attrs[0] = *att;
			*att = prev;
		}
		parser->att_sep = 0;
		parser->sax_state = SAX_STATE_ATT_NAME;
		parser->att_name_start = 0;
		return 0;
	}
	return 1;
}


typedef struct
{
	char *name;
	char *value;
	u32 namelen;
	u8 sep;
} XML_Entity;

static void xml_sax_flush_text(GF_SAXParser *parser)
{
	char *text, c;
	if (!parser->text_start || parser->init_state || !parser->sax_text_content) return;

	assert(parser->text_start < parser->text_end);

	c = parser->buffer[parser->text_end-1];
	parser->buffer[parser->text_end-1] = 0;
	text = parser->buffer + parser->text_start-1;

	/*solve XML built-in entities*/
	if (strchr(text, '&') && strchr(text, ';')) {
		char *xml_text = xml_translate_xml_string(text);
		if (xml_text) {
			parser->sax_text_content(parser->sax_cbck, xml_text, (parser->sax_state==SAX_STATE_CDATA) ? 1 : 0);
			gf_free(xml_text);
		}
	} else {
		parser->sax_text_content(parser->sax_cbck, text, (parser->sax_state==SAX_STATE_CDATA) ? 1 : 0);
	}
	parser->buffer[parser->text_end-1] = c;
	parser->text_start = parser->text_end = 0;
}

static void xml_sax_store_text(GF_SAXParser *parser, u32 txt_len)
{
	if (!txt_len) return;

	if (!parser->text_start) {
		parser->text_start = parser->current_pos + 1;
		parser->text_end = parser->text_start + txt_len;
		parser->current_pos += txt_len;
		assert(parser->current_pos <= parser->line_size);
		return;
	}
	/*contiguous text*/
	if (parser->text_end && (parser->text_end-1 == parser->current_pos)) {
		parser->text_end += txt_len;
		parser->current_pos += txt_len;
		assert(parser->current_pos <= parser->line_size);
		return;
	}
	/*need to flush*/
	xml_sax_flush_text(parser);

	parser->text_start = parser->current_pos + 1;
	parser->text_end = parser->text_start + txt_len;
	parser->current_pos += txt_len;
	assert(parser->current_pos <= parser->line_size);
}

static char *xml_get_current_text(GF_SAXParser *parser)
{
	char *text, c;
	if (!parser->text_start) return NULL;

	c = parser->buffer[parser->text_end-1];
	parser->buffer[parser->text_end-1] = 0;
	text = gf_strdup(parser->buffer + parser->text_start-1);
	parser->buffer[parser->text_end-1] = c;
	parser->text_start = parser->text_end = 0;
	return text;
}

static void xml_sax_skip_doctype(GF_SAXParser *parser)
{
	while (parser->current_pos < parser->line_size) {
		if (parser->buffer[parser->current_pos]=='>') {
			parser->sax_state = SAX_STATE_ELEMENT;
			parser->current_pos++;
			xml_sax_swap(parser);
			return;
		}
		parser->current_pos++;
	}
}

static void xml_sax_skip_xml_proc(GF_SAXParser *parser)
{
	while (parser->current_pos + 1 < parser->line_size) {
		if ((parser->buffer[parser->current_pos]=='?') && (parser->buffer[parser->current_pos+1]=='>')) {
			parser->sax_state = SAX_STATE_ELEMENT;
			parser->current_pos++;
			xml_sax_swap(parser);
			return;
		}
		parser->current_pos++;
	}
}


static void xml_sax_parse_entity(GF_SAXParser *parser)
{
	char szName[1024];
	u32 i = 0;
	XML_Entity *ent = (XML_Entity *)gf_list_last(parser->entities);
	char *skip_chars = " \t\n\r";
	i=0;
	if (ent && ent->value) ent = NULL;
	if (ent) skip_chars = NULL;

	while (parser->current_pos+i < parser->line_size) {
		u8 c = parser->buffer[parser->current_pos+i];
		if (skip_chars && strchr(skip_chars, c)) {
			if (c=='\n') parser->line++;
			parser->current_pos++;
			continue;
		}
		if (!ent && (c=='%')) {
			parser->current_pos+=i+1;
			parser->sax_state = SAX_STATE_SKIP_DOCTYPE;
			return;
		}
		else if (!ent && ((c=='\"') || (c=='\'')) ) {
			szName[i] = 0;
			GF_SAFEALLOC(ent, XML_Entity);
			ent->name = gf_strdup(szName);
			ent->namelen = strlen(ent->name);
			ent->sep = c;
			parser->current_pos += 1+i;
			assert(parser->current_pos < parser->line_size);
			xml_sax_swap(parser);
			i=0;
			gf_list_add(parser->entities, ent);
			skip_chars = NULL;
		} else if (ent && c==ent->sep) {
			xml_sax_store_text(parser, i);

			ent->value = xml_get_current_text(parser);
			if (!ent->value) ent->value = gf_strdup("");

			parser->current_pos += 1;
			assert(parser->current_pos < parser->line_size);
			xml_sax_swap(parser);
			parser->sax_state = SAX_STATE_SKIP_DOCTYPE;
			return;
		} else if (!ent) {
			szName[i] = c;
			i++;
		} else {
			i++;
		}
	}
	xml_sax_store_text(parser, i);
}

static void xml_sax_cdata(GF_SAXParser *parser)
{
	char *cd_end = strstr(parser->buffer + parser->current_pos, "]]>");
	if (!cd_end) {
		xml_sax_store_text(parser, parser->line_size - parser->current_pos);
	} else {
		u32 size = cd_end - (parser->buffer + parser->current_pos);
		xml_sax_store_text(parser, size);
		xml_sax_flush_text(parser);
		parser->current_pos += 3;
		assert(parser->current_pos <= parser->line_size);
		parser->sax_state = SAX_STATE_TEXT_CONTENT;
	}
}

static Bool xml_sax_parse_comments(GF_SAXParser *parser)
{
	char *end = strstr(parser->buffer + parser->current_pos, "-->");
	if (!end) {
		if (parser->line_size>3)
			parser->current_pos = parser->line_size-3;
		xml_sax_swap(parser);
		return 0;
	}

	parser->current_pos += 3 + (u32) (end - (parser->buffer + parser->current_pos) );
	assert(parser->current_pos <= parser->line_size);
	parser->sax_state = SAX_STATE_TEXT_CONTENT;
	parser->text_start = parser->text_end = 0;
	xml_sax_swap(parser);
	return 1;
}



static GF_Err xml_sax_parse(GF_SAXParser *parser, Bool force_parse)
{
	u32 i = 0;
	Bool is_text, is_end;
	u8 c;
	char *elt, sep;
	u32 cdata_sep;

	is_text = 0;
	while (parser->current_pos<parser->line_size) {
		if (!force_parse && parser->suspended) goto exit;

restart:
		is_text = 0;
		switch (parser->sax_state) {
		/*load an XML element*/
		case SAX_STATE_TEXT_CONTENT:
			is_text = 1;
		case SAX_STATE_ELEMENT:
			elt = NULL;
			i=0;
			while ((c = parser->buffer[parser->current_pos+i]) !='<') {
				if ((parser->init_state==2) && (c ==']')) {
					parser->sax_state = SAX_STATE_ATT_NAME;
					parser->current_pos+=i+1;
					goto restart;
				}
				i++;
				if (c=='\n') parser->line++;
				if (parser->current_pos+i==parser->line_size) goto exit;
			}
			if (is_text && i) {
				xml_sax_store_text(parser, i);
				is_text = 0;
				parser->sax_state = SAX_STATE_ELEMENT;
			} else if (i) {
				parser->current_pos += i;
				assert(parser->current_pos < parser->line_size);
			}
			is_end = 0;
			i = 0;
			cdata_sep = 0;
			while (1) {
				char c = parser->buffer[parser->current_pos+1+i];
    			if (!strncmp(parser->buffer+parser->current_pos+1+i, "!--", 3)) { 
				    parser->sax_state = SAX_STATE_COMMENT;
                    i += 3;
                    break;
		        }
				if (!c) {
					i = 0;
					goto exit;
				}
				if ((c=='\t') || (c=='\r') || (c==' ') ) {
					if (i) break;
					else parser->current_pos++;
				}
				else if (c=='\n') {
					parser->line++;
					if (i) break;
					else parser->current_pos++;
				}
				else if (c=='>') break;
				else if (c=='=') break;
				else if (c=='[') {
					i++;
					if (!cdata_sep) cdata_sep = 1;
					else {
						break;
					}
				}
				else if (c=='/') {
					is_end = !i ? 1 : 2;
					i++;
				} else if (c=='<') {
                    if (parser->sax_state != SAX_STATE_COMMENT) {
                        parser->sax_state = SAX_STATE_SYNTAX_ERROR;
					    return GF_CORRUPTED_DATA;
                    }
				} else {
					i++;
				}
{'fraction_non_alphanumeric': 0.167749284987154, 'fraction_numerical': 0.014518396432207087, 'mean_word_length': 3.492486933797909, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 768, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 103}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3357793', 'n_tokens_mistral': 18740, 'n_tokens_neox': 15976, 'n_words': 3199}