<commit_before>#include "QCodeEditor.h"
#include "xo/system/system_tools.h"
#include "xo/system/assert.h"
#include "xo/string/string_tools.h"
#include "xo/system/log.h"
#include <QTextStream>
#include <QMessageBox>
#include <QFileDialog>
#include "xo/serialization/serialize.h"

QCodeEditor::QCodeEditor( QWidget* parent ) :
QWidget( parent ),
textChangedFlag( false )
{
	QVBoxLayout* verticalLayout = new QVBoxLayout( this );
	verticalLayout->setContentsMargins( 0, 0, 0, 0 );
	setLayout( verticalLayout );
	textEdit = new QCodeTextEdit( this );

	QFont font;
	font.setFamily( QStringLiteral( "Consolas" ) );
	font.setPointSize( 9 );
	textEdit->setFont( font );
	textEdit->setLineWrapMode( QPlainTextEdit::NoWrap );
	textEdit->setTabStopWidth( 16 );
	textEdit->setWordWrapMode( QTextOption::NoWrap );
	verticalLayout->addWidget( textEdit );

	connect( textEdit, SIGNAL( textChanged() ), this, SLOT( textEditChanged() ) );
}

QCodeEditor::~QCodeEditor()
{}

QString QCodeEditor::getPlainText() const
{
	return textEdit->toPlainText();
}

void QCodeEditor::open( const QString& filename )
{
	QCodeSyntaxHighlighter* xmlSyntaxHighlighter = new QCodeSyntaxHighlighter( textEdit->document(), QCodeSyntaxHighlighter::detectLanguage( filename ) );

	QFile f( filename );
	if ( f.open( QFile::ReadOnly | QFile::Text ) )
	{
		QTextStream str( &f );
		QString data = str.readAll();
		textEdit->setPlainText( data );
		fileName = filename;
		textChangedFlag = false;
	}
	else xo_error( "Could not open " + filename.toStdString() );
}

void QCodeEditor::openDialog( const QString& folder, const QString& fileTypes )
{
	auto fn = QFileDialog::getOpenFileName( this, "Open File", folder, fileTypes );
	if ( !fn.isEmpty() )
		open( fn );
}

void QCodeEditor::save()
{
	QFile file( fileName );
	if ( !file.open( QIODevice::WriteOnly ) )
	{
		QMessageBox::critical( this, "Error writing file", "Could not open file " + fileName );
		return;
	}
	else
	{
		QTextStream stream( &file );
		stream << textEdit->toPlainText();
		stream.flush();
		file.close();
		textChangedFlag = false;
	}
}

void QCodeEditor::saveAs( const QString& fn )
{
	if ( getFileFormat( fn ) != getFileFormat( fileName ) )
	{
		std::stringstream stri( textEdit->toPlainText().toStdString() );
		xo::prop_node pn = xo::make_serializer( getFileFormat( fileName ) )->read_stream( stri );
		std::stringstream stro;
		xo::make_serializer( getFileFormat( fn ) )->write_stream( stro, pn );

		QCodeSyntaxHighlighter* xmlSyntaxHighlighter = new QCodeSyntaxHighlighter( textEdit->document(), QCodeSyntaxHighlighter::detectLanguage( fn ) );
		textEdit->setPlainText( QString( stro.str().c_str() ) );
	}

	fileName = fn;
	save();
}

QString QCodeEditor::getTitle()
{
	return QFileInfo( fileName ).fileName() + ( hasTextChanged() ? "*" : "" );
}

void QCodeEditor::textEditChanged()
{
	if ( !textChangedFlag )
	{
		textChangedFlag = true;
		emit textChanged();
	}
}

std::string QCodeEditor::getFileFormat( const QString& filename ) const
{
	return xo::path( filename.toStdString() ).extension().string();
}

//
// BasicXMLSyntaxHighlighter
//

QCodeSyntaxHighlighter::QCodeSyntaxHighlighter( QObject* parent, Language l ) : QSyntaxHighlighter( parent )
{
	setLanguage( l );
}

QCodeSyntaxHighlighter::QCodeSyntaxHighlighter( QTextDocument* parent, Language l ) : QSyntaxHighlighter( parent )
{
	setLanguage( l );
}

void QCodeSyntaxHighlighter::highlightBlock( const QString &text )
{
	if ( language == XML )
	{
		// Special treatment for xml element regex as we use captured text to emulate lookbehind
		int xmlElementIndex = m_xmlElementRegex.indexIn( text );
		while ( xmlElementIndex >= 0 )
		{
			int matchedPos = m_xmlElementRegex.pos( 1 );
			int matchedLength = m_xmlElementRegex.cap( 1 ).length();
			setFormat( matchedPos, matchedLength, m_ElementFormat );
			xmlElementIndex = m_xmlElementRegex.indexIn( text, matchedPos + matchedLength );
		}
		highlightByRegex( m_NumberFormat, m_NumberRegex, text );
		highlightByRegex( m_AttributeFormat, m_xmlAttributeRegex, text );
	}
	else
	{
		highlightByRegex( m_NumberFormat, m_NumberRegex, text );
		highlightByRegex( m_AttributeFormat, m_xmlAttributeRegex, text );
		highlightByRegex( m_ElementFormat, m_xmlElementRegex, text );
	}

	// Highlight xml keywords *after* xml elements to fix any occasional / captured into the enclosing element
	for ( auto& regex : m_xmlKeywordRegexes )
		highlightByRegex( m_KeywordFormat, regex, text );

	highlightByRegex( m_ValueFormat, m_xmlValueRegex, text );
	highlightByRegex( m_SpecialFormat, m_SpecialRegex, text );

	if ( language == ZML )
	{
		int i = m_xmlCommentRegex.indexIn( text );
		while  ( i >= 0 )
		{
			int quotes_before = 0;
			for ( int x = 0; x <= i; ++x )
				quotes_before += int( text[ x ] == '\"' );
			if ( quotes_before % 2 == 0 )
			{
				setFormat( i, m_xmlCommentRegex.matchedLength(), m_CommentFormat );
				break;
			}
			else i = m_xmlCommentRegex.indexIn( text, i + 1 );
		}
	}
	else 
	{
		highlightByRegex( m_CommentFormat, m_xmlCommentRegex, text );
	}
}

void QCodeSyntaxHighlighter::highlightByRegex( const QTextCharFormat & format, const QRegExp & regex, const QString & text )
{
	int index = regex.indexIn( text );
	while ( index >= 0 )
	{
		int matchedLength = regex.matchedLength();
		setFormat( index, matchedLength, format );
		index = regex.indexIn( text, index + matchedLength );
	}
}

void QCodeSyntaxHighlighter::setRegexes()
{
	switch ( language )
	{
	case XML:
		m_xmlElementRegex.setPattern( "<[\\s]*[/]?[\\s]*([^\\n]\\w*)(?=[\\s/>])" );
		m_xmlAttributeRegex.setPattern( "\\w+(?=\\=)" );
		m_xmlValueRegex.setPattern( "\"[^\\n\"]+\"(?=[\\s/>])" );
		m_xmlCommentRegex.setPattern( "<!--[^\\n]*-->" );
		m_SpecialRegex.setPattern( "/.^/" );
		m_xmlKeywordRegexes = QList<QRegExp>() << QRegExp( "<\\?" ) << QRegExp( "/>" ) << QRegExp( ">" ) << QRegExp( "<" ) << QRegExp( "</" ) << QRegExp( "\\?>" );
		break;
	case ZML:
		m_xmlElementRegex.setPattern( "\\w+\\s*\\=?\\s*[\\{\\[]" );
		m_xmlAttributeRegex.setPattern( "\\w+\\s*(\\=)" );
		m_xmlValueRegex.setPattern( "\"[^\\n\"]*\"" );
		m_xmlCommentRegex.setPattern( "(#\\s|;)[^\\n]*" );
		m_SpecialRegex.setPattern( "#\\w+" );
		m_xmlKeywordRegexes = QList<QRegExp>() << QRegExp( "\\{" ) << QRegExp( "\\}" ) << QRegExp( "\\[" ) << QRegExp( "\\]" ) << QRegExp( "\\=" );
		break;
	default:
		xo_error( "Unsupported language" );
	}

	m_NumberRegex.setPattern( "\\b([-+]?[\\.\\d]+)" );
}

void QCodeSyntaxHighlighter::setFormats()
{
	m_KeywordFormat.setForeground( Qt::darkGray );
	m_ElementFormat.setForeground( Qt::darkBlue );
	m_ElementFormat.setFontWeight( QFont::Bold );
	m_AttributeFormat.setForeground( Qt::darkBlue );
	//m_AttributeFormat.setFontWeight( QFont::Bold );<commit_msg>added // comment highlighting in zml<commit_after>#include "QCodeEditor.h"
#include "xo/system/system_tools.h"
#include "xo/system/assert.h"
#include "xo/string/string_tools.h"
#include "xo/system/log.h"
#include <QTextStream>
#include <QMessageBox>
#include <QFileDialog>
#include "xo/serialization/serialize.h"

QCodeEditor::QCodeEditor( QWidget* parent ) :
QWidget( parent ),
textChangedFlag( false )
{
	QVBoxLayout* verticalLayout = new QVBoxLayout( this );
	verticalLayout->setContentsMargins( 0, 0, 0, 0 );
	setLayout( verticalLayout );
	textEdit = new QCodeTextEdit( this );

	QFont font;
	font.setFamily( QStringLiteral( "Consolas" ) );
	font.setPointSize( 9 );
	textEdit->setFont( font );
	textEdit->setLineWrapMode( QPlainTextEdit::NoWrap );
	textEdit->setTabStopWidth( 16 );
	textEdit->setWordWrapMode( QTextOption::NoWrap );
	verticalLayout->addWidget( textEdit );

	connect( textEdit, SIGNAL( textChanged() ), this, SLOT( textEditChanged() ) );
}

QCodeEditor::~QCodeEditor()
{}

QString QCodeEditor::getPlainText() const
{
	return textEdit->toPlainText();
}

void QCodeEditor::open( const QString& filename )
{
	QCodeSyntaxHighlighter* xmlSyntaxHighlighter = new QCodeSyntaxHighlighter( textEdit->document(), QCodeSyntaxHighlighter::detectLanguage( filename ) );

	QFile f( filename );
	if ( f.open( QFile::ReadOnly | QFile::Text ) )
	{
		QTextStream str( &f );
		QString data = str.readAll();
		textEdit->setPlainText( data );
		fileName = filename;
		textChangedFlag = false;
	}
	else xo_error( "Could not open " + filename.toStdString() );
}

void QCodeEditor::openDialog( const QString& folder, const QString& fileTypes )
{
	auto fn = QFileDialog::getOpenFileName( this, "Open File", folder, fileTypes );
	if ( !fn.isEmpty() )
		open( fn );
}

void QCodeEditor::save()
{
	QFile file( fileName );
	if ( !file.open( QIODevice::WriteOnly ) )
	{
		QMessageBox::critical( this, "Error writing file", "Could not open file " + fileName );
		return;
	}
	else
	{
		QTextStream stream( &file );
		stream << textEdit->toPlainText();
		stream.flush();
		file.close();
		textChangedFlag = false;
	}
}

void QCodeEditor::saveAs( const QString& fn )
{
	if ( getFileFormat( fn ) != getFileFormat( fileName ) )
	{
		std::stringstream stri( textEdit->toPlainText().toStdString() );
		xo::prop_node pn = xo::make_serializer( getFileFormat( fileName ) )->read_stream( stri );
		std::stringstream stro;
		xo::make_serializer( getFileFormat( fn ) )->write_stream( stro, pn );

		QCodeSyntaxHighlighter* xmlSyntaxHighlighter = new QCodeSyntaxHighlighter( textEdit->document(), QCodeSyntaxHighlighter::detectLanguage( fn ) );
		textEdit->setPlainText( QString( stro.str().c_str() ) );
	}

	fileName = fn;
	save();
}

QString QCodeEditor::getTitle()
{
	return QFileInfo( fileName ).fileName() + ( hasTextChanged() ? "*" : "" );
}

void QCodeEditor::textEditChanged()
{
	if ( !textChangedFlag )
	{
		textChangedFlag = true;
		emit textChanged();
	}
}

std::string QCodeEditor::getFileFormat( const QString& filename ) const
{
	return xo::path( filename.toStdString() ).extension().string();
}

//
// BasicXMLSyntaxHighlighter
//

QCodeSyntaxHighlighter::QCodeSyntaxHighlighter( QObject* parent, Language l ) : QSyntaxHighlighter( parent )
{
	setLanguage( l );
}

QCodeSyntaxHighlighter::QCodeSyntaxHighlighter( QTextDocument* parent, Language l ) : QSyntaxHighlighter( parent )
{
	setLanguage( l );
}

void QCodeSyntaxHighlighter::highlightBlock( const QString &text )
{
	if ( language == XML )
	{
		// Special treatment for xml element regex as we use captured text to emulate lookbehind
		int xmlElementIndex = m_xmlElementRegex.indexIn( text );
		while ( xmlElementIndex >= 0 )
		{
			int matchedPos = m_xmlElementRegex.pos( 1 );
			int matchedLength = m_xmlElementRegex.cap( 1 ).length();
			setFormat( matchedPos, matchedLength, m_ElementFormat );
			xmlElementIndex = m_xmlElementRegex.indexIn( text, matchedPos + matchedLength );
		}
		highlightByRegex( m_NumberFormat, m_NumberRegex, text );
		highlightByRegex( m_AttributeFormat, m_xmlAttributeRegex, text );
	}
	else
	{
		highlightByRegex( m_NumberFormat, m_NumberRegex, text );
		highlightByRegex( m_AttributeFormat, m_xmlAttributeRegex, text );
		highlightByRegex( m_ElementFormat, m_xmlElementRegex, text );
	}

	// Highlight xml keywords *after* xml elements to fix any occasional / captured into the enclosing element
	for ( auto& regex : m_xmlKeywordRegexes )
		highlightByRegex( m_KeywordFormat, regex, text );

	highlightByRegex( m_ValueFormat, m_xmlValueRegex, text );
	highlightByRegex( m_SpecialFormat, m_SpecialRegex, text );

	if ( language == ZML )
	{
		int i = m_xmlCommentRegex.indexIn( text );
		while  ( i >= 0 )
		{
			int quotes_before = 0;
			for ( int x = 0; x <= i; ++x )
				quotes_before += int( text[ x ] == '\"' );
			if ( quotes_before % 2 == 0 )
			{
				setFormat( i, m_xmlCommentRegex.matchedLength(), m_CommentFormat );
				break;
			}
			else i = m_xmlCommentRegex.indexIn( text, i + 1 );
		}
	}
	else 
	{
		highlightByRegex( m_CommentFormat, m_xmlCommentRegex, text );
	}
}

void QCodeSyntaxHighlighter::highlightByRegex( const QTextCharFormat & format, const QRegExp & regex, const QString & text )
{
	int index = regex.indexIn( text );
	while ( index >= 0 )
	{
		int matchedLength = regex.matchedLength();
		setFormat( index, matchedLength, format );
		index = regex.indexIn( text, index + matchedLength );
	}
}

void QCodeSyntaxHighlighter::setRegexes()
{
	switch ( language )
	{
	case XML:
		m_xmlElementRegex.setPattern( "<[\\s]*[/]?[\\s]*([^\\n]\\w*)(?=[\\s/>])" );
		m_xmlAttributeRegex.setPattern( "\\w+(?=\\=)" );
		m_xmlValueRegex.setPattern( "\"[^\\n\"]+\"(?=[\\s/>])" );
		m_xmlCommentRegex.setPattern( "<!--[^\\n]*-->" );
		m_SpecialRegex.setPattern( "/.^/" );
		m_xmlKeywordRegexes = QList<QRegExp>() << QRegExp( "<\\?" ) << QRegExp( "/>" ) << QRegExp( ">" ) << QRegExp( "<" ) << QRegExp( "</" ) << QRegExp( "\\?>" );
		break;
	case ZML:
		m_xmlElementRegex.setPattern( "\\w+\\s*\\=?\\s*[\\{\\[]" );
		m_xmlAttributeRegex.setPattern( "\\w+\\s*(\\=)" );
		m_xmlValueRegex.setPattern( "\"[^\\n\"]*\"" );
		m_xmlCommentRegex.setPattern( "(#\\s|;|//)[^\\n]*" );
		m_SpecialRegex.setPattern( "#\\w+" );
		m_xmlKeywordRegexes = QList<QRegExp>() << QRegExp( "\\{" ) << QRegExp( "\\}" ) << QRegExp( "\\[" ) << QRegExp( "\\]" ) << QRegExp( "\\=" );
		break;
	default:
		xo_error( "Unsupported language" );
	}

	m_NumberRegex.setPattern( "\\b([-+]?[\\.\\d]+)" );
}

void QCodeSyntaxHighlighter::setFormats()
{
	m_KeywordFormat.setForeground( Qt::darkGray );
	m_ElementFormat.setForeground( Qt::darkBlue );
	m_ElementFormat.setFontWeight( QFont::Bold );
	m_AttributeFormat.setForeground( Qt::darkBlue );
	//m_AttributeFormat.setFontWeight( QFont::Bold );
{'fraction_non_alphanumeric': 0.162481426448737, 'fraction_numerical': 0.002526002971768202, 'mean_word_length': 4.674957841483979, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 62}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2930916', 'n_tokens_mistral': 4966, 'n_tokens_neox': 4711, 'n_words': 958}