<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: tagtest.cxx,v $
 *
 *  $Revision: 1.18 $
 *
 *  last change: $Author: obo $ $Date: 2006-10-11 09:55:35 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_transex3.hxx"

#ifndef _STRING_HXX
#include <tools/string.hxx>
#endif
#include "tagtest.hxx"

#if OSL_DEBUG_LEVEL > 1
#include <stdio.h>
#endif

#include "gsicheck.hxx"

#define HAS_FLAG( nFlags, nFlag )       ( ( nFlags & nFlag ) != 0 )
#define SET_FLAG( nFlags, nFlag )       ( nFlags |= nFlag )
#define RESET_FLAG( nFlags, nFlag )     ( nFlags &= ~nFlag )    // ~ = Bitweises NOT



TokenInfo::TokenInfo( TokenId pnId, USHORT nP, String paStr, ParserMessageList &rErrorList )
: bClosed(FALSE)
, bCloseTag(FALSE)
, bIsBroken(FALSE)
, bHasBeenFixed(FALSE)
, bDone(FALSE)
, aTokenString( paStr )
, nId( pnId )
, nPos(nP)
{
    if ( nId == TAG_COMMONSTART || nId == TAG_COMMONEND )
        SplitTag( rErrorList );
}

enum tagcheck { TC_START, TC_HAS_TAG_NAME, TC_HAS_PROP_NAME_EQ, TC_HAS_PROP_NAME_EQ_SP, TC_HAS_PROP_NAME_SP, TC_INSIDE_STRING, TC_PROP_FINISHED, TC_CLOSED, TC_CLOSED_SPACE, TC_CLOSETAG, TC_CLOSETAG_HAS_TAG_NAME, TC_FINISHED, TC_ERROR };

/*
                                                      \<  link  href  =  \"text\"  name  =  \"C\"  \>
START               ' ' ->  HAS_TAG_NAME
START               '/' ->  CLOSED
START               '/' ->  CLOSETAG    - no Portion (starting with /)
START               '>' ->  FINISHED
HAS_TAG_NAME        '=' ->  HAS_PROP_NAME_EQ
HAS_TAG_NAME        ' ' ->  HAS_PROP_NAME_SP
HAS_TAG_NAME        '/' ->  CLOSED
HAS_TAG_NAME        '>' ->  FINISHED
HAS_PROP_NAME_SP    '=' ->  HAS_PROP_NAME_EQ
HAS_PROP_NAME_EQ    ' ' ->  HAS_PROP_NAME_EQ_SP
HAS_PROP_NAME_EQ    '"' ->  INSIDE_STRING
HAS_PROP_NAME_EQ_SP '"' ->  INSIDE_STRING
INSIDE_STRING       ' ' ->  INSIDE_STRING
INSIDE_STRING       '=' ->  INSIDE_STRING
INSIDE_STRING       '>' ->  INSIDE_STRING
INSIDE_STRING       '"' ->  PROP_FINISHED
PROP_FINISHED       ' ' ->  HAS_TAG_NAME
PROP_FINISHED       '/' ->  CLOSED
PROP_FINISHED       '>' ->  FINISHED
CLOSED              ' ' ->  CLOSED_SPACE
CLOSED              '>' ->  FINISHED
CLOSED_SPACE        '>' ->  FINISHED

CLOSETAG            ' ' ->  CLOSETAG_HAS_TAG_NAME
CLOSETAG            '>' ->  FINISHED
CLOSETAG_HAS_TAG_NAME  '>' ->  FINISHED

*/
void TokenInfo::SplitTag( ParserMessageList &rErrorList )
{
    USHORT nLastPos = 2;    // skip initial  \<
    USHORT nCheckPos = nLastPos;
    String aDelims( String::CreateFromAscii( " \\=>/" ) );
    String aPortion;
    String aValue;      // store the value of a property
    ByteString aName;   // store the name of a property/tag
    BOOL bCheckName = FALSE;
    BOOL bCheckEmpty = FALSE;
    sal_Unicode cDelim;
    tagcheck aState = TC_START;

    // skip blanks
    while ( nLastPos < aTokenString.Len() && aTokenString.GetChar( nLastPos ) == ' ')
        nLastPos++;

    nCheckPos = aTokenString.SearchChar( aDelims.GetBuffer(), nLastPos );
    while ( nCheckPos != STRING_NOTFOUND && !( aState == TC_FINISHED || aState == TC_ERROR ) )
    {
        aPortion = aTokenString.Copy( nLastPos, nCheckPos-nLastPos );

        if ( aTokenString.GetChar( nCheckPos ) == '\\' )
            nCheckPos++;

        cDelim = aTokenString.GetChar( nCheckPos );
        nCheckPos++;

        switch ( aState )
        {
//            START           ' ' ->  HAS_TAG_NAME
//            START           '/' ->  CLOSED
//            START           '>' ->  FINISHED
            case TC_START:
                aTagName = aPortion;
                switch ( cDelim )
                {
                    case ' ':  aState = TC_HAS_TAG_NAME;
                               bCheckName = TRUE;
                               break;
                    case '/':
                        {
                            if ( aPortion.Len() == 0 )
                            {
                                aState = TC_CLOSETAG;
                            }
                            else
                            {
                                aState = TC_CLOSED;
                                bCheckName = TRUE;
                            }
                        }
                        break;
                    case '>':  aState = TC_FINISHED;
                               bCheckName = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            HAS_TAG_NAME    '=' ->  HAS_PROP_NAME_EQ
//            HAS_TAG_NAME    ' ' ->  HAS_PROP_NAME_SP
//            HAS_TAG_NAME    '/' ->  CLOSED
//            HAS_TAG_NAME    '>' ->  FINISHED
            case TC_HAS_TAG_NAME:
                switch ( cDelim )
                {
                    case '=':  aState = TC_HAS_PROP_NAME_EQ;
                               bCheckName = TRUE;
                               break;
                    case ' ':  aState = TC_HAS_PROP_NAME_SP;
                               bCheckName = TRUE;
                               break;
                    case '/':  aState = TC_CLOSED;
                               bCheckEmpty = TRUE;
                               break;
                    case '>':  aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            HAS_PROP_NAME_SP    '=' ->  HAS_PROP_NAME_EQ
            case TC_HAS_PROP_NAME_SP:
                switch ( cDelim )
                {
                    case '=':  aState = TC_HAS_PROP_NAME_EQ;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            HAS_PROP_NAME_EQ    ' ' ->  HAS_PROP_NAME_EQ_SP
//            HAS_PROP_NAME_EQ    '"' ->  INSIDE_STRING
            case TC_HAS_PROP_NAME_EQ:
                switch ( cDelim )
                {
                    case ' ':  aState = TC_HAS_PROP_NAME_EQ_SP;
                               bCheckEmpty = TRUE;
                               break;
                    case '\"': aState = TC_INSIDE_STRING;
                               bCheckEmpty = TRUE;
                               aValue.Erase();
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            HAS_PROP_NAME_EQ_SP '"' ->  INSIDE_STRING
            case TC_HAS_PROP_NAME_EQ_SP:
                switch ( cDelim )
                {
                    case '\"': aState = TC_INSIDE_STRING;
                               bCheckEmpty = TRUE;
                               aValue.Erase();
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            INSIDE_STRING    *  ->  INSIDE_STRING
//            INSIDE_STRING   '"' ->  PROP_FINISHED
            case TC_INSIDE_STRING:
                switch ( cDelim )
                {
                    case '\"':
                        {
                            aState = TC_PROP_FINISHED;
                            aValue += aPortion;
                            if ( aProperties.find( aName ) == aProperties.end() )
                            {
                                if ( !IsPropertyValueValid( aName, aValue ) )
                                {
                                    rErrorList.AddError( 25, ByteString("Property '").Append(aName).Append("' has invalid value '").Append(ByteString( aValue, RTL_TEXTENCODING_UTF8 )).Append("' "), *this );
                                    bIsBroken = TRUE;
                                }
                                aProperties[ aName ] = aValue;
                            }
                            else
                            {
                                rErrorList.AddError( 25, ByteString("Property '").Append(aName).Append("' defined twice "), *this );
                                bIsBroken = TRUE;
                            }
                        }
                               break;
                    default:
                        {
                            aState = TC_INSIDE_STRING;
                            aValue += aPortion;
                            aValue += cDelim;
                        }
                }
                break;

//            PROP_FINISHED   ' ' ->  HAS_TAG_NAME
//            PROP_FINISHED   '/' ->  CLOSED
//            PROP_FINISHED   '>' ->  FINISHED
            case TC_PROP_FINISHED:
                switch ( cDelim )
                {
                    case ' ': aState = TC_HAS_TAG_NAME;
                               bCheckEmpty = TRUE;
                               break;
                    case '/': aState = TC_CLOSED;
                               bCheckEmpty = TRUE;
                               break;
                    case '>': aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            CLOSED          ' ' ->  CLOSED_SPACE
//            CLOSED          '>' ->  FINISHED
            case TC_CLOSED:
                switch ( cDelim )
                {
                    case ' ': aState = TC_CLOSED_SPACE;
                               bCheckEmpty = TRUE;
                               bClosed = TRUE;
                               break;
                    case '>': aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            CLOSED_SPACE    '>' ->  FINISHED
            case TC_CLOSED_SPACE:
                switch ( cDelim )
                {
                    case '>': aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

// CLOSETAG            ' ' ->  CLOSETAG_HAS_TAG_NAME
// CLOSETAG            '>' ->  FINISHED
            case TC_CLOSETAG:
                bCloseTag = TRUE;
                switch ( cDelim )
                {
                    case ' ': aState = TC_CLOSETAG_HAS_TAG_NAME;
                               aTagName = aPortion;
                               bCheckName = TRUE;
                               break;
                    case '>': aState = TC_FINISHED;
                               aTagName = aPortion;
                               bCheckName = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

// CLOSETAG_HAS_TAG_NAME       '>' ->  FINISHED
            case TC_CLOSETAG_HAS_TAG_NAME:
                switch ( cDelim )
                {
                    case '>': aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;


            default: rErrorList.AddError( 99, "Internal error Parsing Tag ", *this );
                     bIsBroken = TRUE;

        }

        if ( bCheckName )
        {
            if ( aPortion.Len() == 0 )
            {
                rErrorList.AddError( 25, "Tag/Property name missing ", *this );
                bIsBroken = TRUE;
            }
            else
            {
                aName = ByteString( aPortion, RTL_TEXTENCODING_UTF8 );
                // "a-zA-Z_-.0-9"
                xub_StrLen nCount;
                BOOL bBroken = FALSE;
                const sal_Char* aBuf = aName.GetBuffer();
                for ( nCount = 0 ; !bBroken && nCount < aName.Len() ; nCount++ )
                {
                    bBroken = ! (   ( aBuf[nCount] >= 'a' && aBuf[nCount] <= 'z' )
                                ||( aBuf[nCount] >= 'A' && aBuf[nCount] <= 'Z' )
                                ||( aBuf[nCount] >= '0' && aBuf[nCount] <= '9' )
                                ||( aBuf[nCount] == '_' )
                                ||( aBuf[nCount] == '-' )
                                ||( aBuf[nCount] == '.' )
                                );
                }

                if ( bBroken )
                {
                    rErrorList.AddError( 25, "Found illegal character in Tag/Property name ", *this );
                    bIsBroken = TRUE;
                }
            }

            bCheckName = FALSE;
        }

        if ( bCheckEmpty )
        {
            if ( aPortion.Len() )
            {
                rErrorList.AddError( 25, ByteString("Found displaced characters '").Append(ByteString( aPortion, RTL_TEXTENCODING_UTF8 )).Append("' in Tag "), *this );
                bIsBroken = TRUE;
            }
            bCheckEmpty = FALSE;
        }


        nLastPos = nCheckPos;

        // skip further blanks
        if ( cDelim == ' ' && aState != TC_INSIDE_STRING )
            while ( nLastPos < aTokenString.Len() && aTokenString.GetChar( nLastPos ) == ' ')
                nLastPos++;

        nCheckPos = aTokenString.SearchChar( aDelims.GetBuffer(), nLastPos );
    }
    if ( aState != TC_FINISHED )
    {
        rErrorList.AddError( 25, "Parsing error in Tag ", *this );
        bIsBroken = TRUE;
    }
}

BOOL TokenInfo::IsPropertyRelevant( const ByteString &aName, const String &aValue ) const
{
    if ( aTagName.EqualsAscii( "alt" ) && aName.Equals( "xml-lang" ) )
        return FALSE;
    if ( aTagName.EqualsAscii( "ahelp" ) && aName.Equals( "visibility" ) && aValue.EqualsAscii("visible") )
        return FALSE;
    if ( aTagName.EqualsAscii( "image" ) && (aName.Equals( "width" ) || aName.Equals( "height" )) )
        return FALSE;

    return TRUE;
}

BOOL TokenInfo::IsPropertyValueValid( const ByteString &aName, const String &aValue ) const
{
/*  removed due to i56740
    if ( aTagName.EqualsAscii( "switchinline" ) && aName.Equals( "select" ) )
    {
        return aValue.EqualsAscii("sys") ||
               aValue.EqualsAscii("appl") ||
               aValue.EqualsAscii("distrib");
    } */
    if ( aTagName.EqualsAscii( "caseinline" ) && aName.Equals( "select" ) )
    {
        return /*!aValue.EqualsAscii("OS2") &&  removed due to i56740 */
               !aValue.EqualsAscii("");
    }

    // we don't know any better so we assume it to be OK
    return TRUE;
}

BOOL TokenInfo::IsPropertyInvariant( const ByteString &aName ) const
{
    if ( aTagName.EqualsAscii( "link" ) && aName.Equals( "name" ) )
        return FALSE;
    return TRUE;
}

BOOL TokenInfo::IsPropertyFixable( const ByteString &aName ) const
{
    // name everything that is allowed to be fixed automatically here
    if ( aTagName.EqualsAscii( "ahelp" ) && aName.Equals( "hid" )
      || aTagName.EqualsAscii( "link" ) && aName.Equals( "href" )
      || aTagName.EqualsAscii( "alt" ) && aName.Equals( "id" )
      || aTagName.EqualsAscii( "variable" ) && aName.Equals( "id" )
      || aTagName.EqualsAscii( "image" ) && aName.Equals( "src" )
      || aTagName.EqualsAscii( "image" ) && aName.Equals( "id" ) )
        return TRUE;
    return FALSE;
}

BOOL TokenInfo::MatchesTranslation( TokenInfo& rInfo, BOOL bGenErrors, ParserMessageList &rErrorList, BOOL bFixTags ) const
{
    // check if tags are equal
    // check if all existing properties are in the translation as well and
    // wether they have a matching content (the same in most cases)

    if ( nId != rInfo.nId )
        return FALSE;

    if ( !aTagName.Equals( rInfo.aTagName ) )
        return FALSE;

    // If one of the tags has formating errors already it does make no sense to check here, so return right away
    if ( bGenErrors && ( bIsBroken || rInfo.bIsBroken ) )
        return TRUE;

    StringHashMap::const_iterator iProp;
    for( iProp = aProperties.begin() ; iProp != aProperties.end(); ++iProp )
    {
        if ( rInfo.aProperties.find( iProp->first ) != rInfo.aProperties.end() )
        {
            if ( IsPropertyRelevant( iProp->first, iProp->second ) || IsPropertyRelevant( iProp->first, rInfo.aProperties.find( iProp->first )->second ) )
            {
                if ( IsPropertyInvariant( iProp->first ) )
                {
                    if ( !rInfo.aProperties.find( iProp->first )->second.Equals( iProp->second ) )
                    {
                        if ( bGenErrors )
                        {
                            if ( bFixTags && IsPropertyFixable( iProp->first ) )
                            {
                                rInfo.aProperties.find( iProp->first )->second = iProp->second;
                                rInfo.SetHasBeenFixed();<commit_msg>INTEGRATION: CWS gh13 (1.18.14); FILE MERGED 2007/05/03 09:54:40 gh 1.18.14.1: do not check external links for equality<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: tagtest.cxx,v $
 *
 *  $Revision: 1.19 $
 *
 *  last change: $Author: kz $ $Date: 2007-05-11 09:12:34 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_transex3.hxx"

#ifndef _STRING_HXX
#include <tools/string.hxx>
#endif
#include "tagtest.hxx"

#if OSL_DEBUG_LEVEL > 1
#include <stdio.h>
#endif

#include "gsicheck.hxx"

#define HAS_FLAG( nFlags, nFlag )       ( ( nFlags & nFlag ) != 0 )
#define SET_FLAG( nFlags, nFlag )       ( nFlags |= nFlag )
#define RESET_FLAG( nFlags, nFlag )     ( nFlags &= ~nFlag )    // ~ = Bitweises NOT



TokenInfo::TokenInfo( TokenId pnId, USHORT nP, String paStr, ParserMessageList &rErrorList )
: bClosed(FALSE)
, bCloseTag(FALSE)
, bIsBroken(FALSE)
, bHasBeenFixed(FALSE)
, bDone(FALSE)
, aTokenString( paStr )
, nId( pnId )
, nPos(nP)
{
    if ( nId == TAG_COMMONSTART || nId == TAG_COMMONEND )
        SplitTag( rErrorList );
}

enum tagcheck { TC_START, TC_HAS_TAG_NAME, TC_HAS_PROP_NAME_EQ, TC_HAS_PROP_NAME_EQ_SP, TC_HAS_PROP_NAME_SP, TC_INSIDE_STRING, TC_PROP_FINISHED, TC_CLOSED, TC_CLOSED_SPACE, TC_CLOSETAG, TC_CLOSETAG_HAS_TAG_NAME, TC_FINISHED, TC_ERROR };

/*
                                                      \<  link  href  =  \"text\"  name  =  \"C\"  \>
START               ' ' ->  HAS_TAG_NAME
START               '/' ->  CLOSED
START               '/' ->  CLOSETAG    - no Portion (starting with /)
START               '>' ->  FINISHED
HAS_TAG_NAME        '=' ->  HAS_PROP_NAME_EQ
HAS_TAG_NAME        ' ' ->  HAS_PROP_NAME_SP
HAS_TAG_NAME        '/' ->  CLOSED
HAS_TAG_NAME        '>' ->  FINISHED
HAS_PROP_NAME_SP    '=' ->  HAS_PROP_NAME_EQ
HAS_PROP_NAME_EQ    ' ' ->  HAS_PROP_NAME_EQ_SP
HAS_PROP_NAME_EQ    '"' ->  INSIDE_STRING
HAS_PROP_NAME_EQ_SP '"' ->  INSIDE_STRING
INSIDE_STRING       ' ' ->  INSIDE_STRING
INSIDE_STRING       '=' ->  INSIDE_STRING
INSIDE_STRING       '>' ->  INSIDE_STRING
INSIDE_STRING       '"' ->  PROP_FINISHED
PROP_FINISHED       ' ' ->  HAS_TAG_NAME
PROP_FINISHED       '/' ->  CLOSED
PROP_FINISHED       '>' ->  FINISHED
CLOSED              ' ' ->  CLOSED_SPACE
CLOSED              '>' ->  FINISHED
CLOSED_SPACE        '>' ->  FINISHED

CLOSETAG            ' ' ->  CLOSETAG_HAS_TAG_NAME
CLOSETAG            '>' ->  FINISHED
CLOSETAG_HAS_TAG_NAME  '>' ->  FINISHED

*/
void TokenInfo::SplitTag( ParserMessageList &rErrorList )
{
    USHORT nLastPos = 2;    // skip initial  \<
    USHORT nCheckPos = nLastPos;
    String aDelims( String::CreateFromAscii( " \\=>/" ) );
    String aPortion;
    String aValue;      // store the value of a property
    ByteString aName;   // store the name of a property/tag
    BOOL bCheckName = FALSE;
    BOOL bCheckEmpty = FALSE;
    sal_Unicode cDelim;
    tagcheck aState = TC_START;

    // skip blanks
    while ( nLastPos < aTokenString.Len() && aTokenString.GetChar( nLastPos ) == ' ')
        nLastPos++;

    nCheckPos = aTokenString.SearchChar( aDelims.GetBuffer(), nLastPos );
    while ( nCheckPos != STRING_NOTFOUND && !( aState == TC_FINISHED || aState == TC_ERROR ) )
    {
        aPortion = aTokenString.Copy( nLastPos, nCheckPos-nLastPos );

        if ( aTokenString.GetChar( nCheckPos ) == '\\' )
            nCheckPos++;

        cDelim = aTokenString.GetChar( nCheckPos );
        nCheckPos++;

        switch ( aState )
        {
//            START           ' ' ->  HAS_TAG_NAME
//            START           '/' ->  CLOSED
//            START           '>' ->  FINISHED
            case TC_START:
                aTagName = aPortion;
                switch ( cDelim )
                {
                    case ' ':  aState = TC_HAS_TAG_NAME;
                               bCheckName = TRUE;
                               break;
                    case '/':
                        {
                            if ( aPortion.Len() == 0 )
                            {
                                aState = TC_CLOSETAG;
                            }
                            else
                            {
                                aState = TC_CLOSED;
                                bCheckName = TRUE;
                            }
                        }
                        break;
                    case '>':  aState = TC_FINISHED;
                               bCheckName = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            HAS_TAG_NAME    '=' ->  HAS_PROP_NAME_EQ
//            HAS_TAG_NAME    ' ' ->  HAS_PROP_NAME_SP
//            HAS_TAG_NAME    '/' ->  CLOSED
//            HAS_TAG_NAME    '>' ->  FINISHED
            case TC_HAS_TAG_NAME:
                switch ( cDelim )
                {
                    case '=':  aState = TC_HAS_PROP_NAME_EQ;
                               bCheckName = TRUE;
                               break;
                    case ' ':  aState = TC_HAS_PROP_NAME_SP;
                               bCheckName = TRUE;
                               break;
                    case '/':  aState = TC_CLOSED;
                               bCheckEmpty = TRUE;
                               break;
                    case '>':  aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            HAS_PROP_NAME_SP    '=' ->  HAS_PROP_NAME_EQ
            case TC_HAS_PROP_NAME_SP:
                switch ( cDelim )
                {
                    case '=':  aState = TC_HAS_PROP_NAME_EQ;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            HAS_PROP_NAME_EQ    ' ' ->  HAS_PROP_NAME_EQ_SP
//            HAS_PROP_NAME_EQ    '"' ->  INSIDE_STRING
            case TC_HAS_PROP_NAME_EQ:
                switch ( cDelim )
                {
                    case ' ':  aState = TC_HAS_PROP_NAME_EQ_SP;
                               bCheckEmpty = TRUE;
                               break;
                    case '\"': aState = TC_INSIDE_STRING;
                               bCheckEmpty = TRUE;
                               aValue.Erase();
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            HAS_PROP_NAME_EQ_SP '"' ->  INSIDE_STRING
            case TC_HAS_PROP_NAME_EQ_SP:
                switch ( cDelim )
                {
                    case '\"': aState = TC_INSIDE_STRING;
                               bCheckEmpty = TRUE;
                               aValue.Erase();
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            INSIDE_STRING    *  ->  INSIDE_STRING
//            INSIDE_STRING   '"' ->  PROP_FINISHED
            case TC_INSIDE_STRING:
                switch ( cDelim )
                {
                    case '\"':
                        {
                            aState = TC_PROP_FINISHED;
                            aValue += aPortion;
                            if ( aProperties.find( aName ) == aProperties.end() )
                            {
                                if ( !IsPropertyValueValid( aName, aValue ) )
                                {
                                    rErrorList.AddError( 25, ByteString("Property '").Append(aName).Append("' has invalid value '").Append(ByteString( aValue, RTL_TEXTENCODING_UTF8 )).Append("' "), *this );
                                    bIsBroken = TRUE;
                                }
                                aProperties[ aName ] = aValue;
                            }
                            else
                            {
                                rErrorList.AddError( 25, ByteString("Property '").Append(aName).Append("' defined twice "), *this );
                                bIsBroken = TRUE;
                            }
                        }
                               break;
                    default:
                        {
                            aState = TC_INSIDE_STRING;
                            aValue += aPortion;
                            aValue += cDelim;
                        }
                }
                break;

//            PROP_FINISHED   ' ' ->  HAS_TAG_NAME
//            PROP_FINISHED   '/' ->  CLOSED
//            PROP_FINISHED   '>' ->  FINISHED
            case TC_PROP_FINISHED:
                switch ( cDelim )
                {
                    case ' ': aState = TC_HAS_TAG_NAME;
                               bCheckEmpty = TRUE;
                               break;
                    case '/': aState = TC_CLOSED;
                               bCheckEmpty = TRUE;
                               break;
                    case '>': aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            CLOSED          ' ' ->  CLOSED_SPACE
//            CLOSED          '>' ->  FINISHED
            case TC_CLOSED:
                switch ( cDelim )
                {
                    case ' ': aState = TC_CLOSED_SPACE;
                               bCheckEmpty = TRUE;
                               bClosed = TRUE;
                               break;
                    case '>': aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

//            CLOSED_SPACE    '>' ->  FINISHED
            case TC_CLOSED_SPACE:
                switch ( cDelim )
                {
                    case '>': aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

// CLOSETAG            ' ' ->  CLOSETAG_HAS_TAG_NAME
// CLOSETAG            '>' ->  FINISHED
            case TC_CLOSETAG:
                bCloseTag = TRUE;
                switch ( cDelim )
                {
                    case ' ': aState = TC_CLOSETAG_HAS_TAG_NAME;
                               aTagName = aPortion;
                               bCheckName = TRUE;
                               break;
                    case '>': aState = TC_FINISHED;
                               aTagName = aPortion;
                               bCheckName = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;

// CLOSETAG_HAS_TAG_NAME       '>' ->  FINISHED
            case TC_CLOSETAG_HAS_TAG_NAME:
                switch ( cDelim )
                {
                    case '>': aState = TC_FINISHED;
                               bCheckEmpty = TRUE;
                               break;
                    default:   aState = TC_ERROR;
                }
                break;


            default: rErrorList.AddError( 99, "Internal error Parsing Tag ", *this );
                     bIsBroken = TRUE;

        }

        if ( bCheckName )
        {
            if ( aPortion.Len() == 0 )
            {
                rErrorList.AddError( 25, "Tag/Property name missing ", *this );
                bIsBroken = TRUE;
            }
            else
            {
                aName = ByteString( aPortion, RTL_TEXTENCODING_UTF8 );
                // "a-zA-Z_-.0-9"
                xub_StrLen nCount;
                BOOL bBroken = FALSE;
                const sal_Char* aBuf = aName.GetBuffer();
                for ( nCount = 0 ; !bBroken && nCount < aName.Len() ; nCount++ )
                {
                    bBroken = ! (   ( aBuf[nCount] >= 'a' && aBuf[nCount] <= 'z' )
                                ||( aBuf[nCount] >= 'A' && aBuf[nCount] <= 'Z' )
                                ||( aBuf[nCount] >= '0' && aBuf[nCount] <= '9' )
                                ||( aBuf[nCount] == '_' )
                                ||( aBuf[nCount] == '-' )
                                ||( aBuf[nCount] == '.' )
                                );
                }

                if ( bBroken )
                {
                    rErrorList.AddError( 25, "Found illegal character in Tag/Property name ", *this );
                    bIsBroken = TRUE;
                }
            }

            bCheckName = FALSE;
        }

        if ( bCheckEmpty )
        {
            if ( aPortion.Len() )
            {
                rErrorList.AddError( 25, ByteString("Found displaced characters '").Append(ByteString( aPortion, RTL_TEXTENCODING_UTF8 )).Append("' in Tag "), *this );
                bIsBroken = TRUE;
            }
            bCheckEmpty = FALSE;
        }


        nLastPos = nCheckPos;

        // skip further blanks
        if ( cDelim == ' ' && aState != TC_INSIDE_STRING )
            while ( nLastPos < aTokenString.Len() && aTokenString.GetChar( nLastPos ) == ' ')
                nLastPos++;

        nCheckPos = aTokenString.SearchChar( aDelims.GetBuffer(), nLastPos );
    }
    if ( aState != TC_FINISHED )
    {
        rErrorList.AddError( 25, "Parsing error in Tag ", *this );
        bIsBroken = TRUE;
    }
}

BOOL TokenInfo::IsPropertyRelevant( const ByteString &aName, const String &aValue ) const
{
    if ( aTagName.EqualsAscii( "alt" ) && aName.Equals( "xml-lang" ) )
        return FALSE;
    if ( aTagName.EqualsAscii( "ahelp" ) && aName.Equals( "visibility" ) && aValue.EqualsAscii("visible") )
        return FALSE;
    if ( aTagName.EqualsAscii( "image" ) && (aName.Equals( "width" ) || aName.Equals( "height" )) )
        return FALSE;

    return TRUE;
}

BOOL TokenInfo::IsPropertyValueValid( const ByteString &aName, const String &aValue ) const
{
/*  removed due to i56740
    if ( aTagName.EqualsAscii( "switchinline" ) && aName.Equals( "select" ) )
    {
        return aValue.EqualsAscii("sys") ||
               aValue.EqualsAscii("appl") ||
               aValue.EqualsAscii("distrib");
    } */
    if ( aTagName.EqualsAscii( "caseinline" ) && aName.Equals( "select" ) )
    {
        return /*!aValue.EqualsAscii("OS2") &&  removed due to i56740 */
               !aValue.EqualsAscii("");
    }

    // we don't know any better so we assume it to be OK
    return TRUE;
}

BOOL TokenInfo::IsPropertyInvariant( const ByteString &aName, const String &aValue ) const
{
    if ( aTagName.EqualsAscii( "link" ) && aName.Equals( "name" ) )
        return FALSE;
    if ( aTagName.EqualsAscii( "link" ) && aName.Equals( "href" ) )
    {   // check for external reference
        if (  aValue.Copy( 0, 5 ).EqualsIgnoreCaseAscii( "http:" )
           || aValue.Copy( 0, 6 ).EqualsIgnoreCaseAscii( "https:" )
           || aValue.Copy( 0, 4 ).EqualsIgnoreCaseAscii( "ftp:" ) )
            return FALSE;
        else
            return TRUE;
    }
    return TRUE;
}

BOOL TokenInfo::IsPropertyFixable( const ByteString &aName ) const
{
    // name everything that is allowed to be fixed automatically here
    if ( aTagName.EqualsAscii( "ahelp" ) && aName.Equals( "hid" )
      || aTagName.EqualsAscii( "link" ) && aName.Equals( "href" )
      || aTagName.EqualsAscii( "alt" ) && aName.Equals( "id" )
      || aTagName.EqualsAscii( "variable" ) && aName.Equals( "id" )
      || aTagName.EqualsAscii( "image" ) && aName.Equals( "src" )
      || aTagName.EqualsAscii( "image" ) && aName.Equals( "id" ) )
        return TRUE;
    return FALSE;
}

BOOL TokenInfo::MatchesTranslation( TokenInfo& rInfo, BOOL bGenErrors, ParserMessageList &rErrorList, BOOL bFixTags ) const
{
    // check if tags are equal
    // check if all existing properties are in the translation as well and
    // wether they have a matching content (the same in most cases)

    if ( nId != rInfo.nId )
        return FALSE;

    if ( !aTagName.Equals( rInfo.aTagName ) )
        return FALSE;

    // If one of the tags has formating errors already it does make no sense to check here, so return right away
    if ( bGenErrors && ( bIsBroken || rInfo.bIsBroken ) )
        return TRUE;

    StringHashMap::const_iterator iProp;
    for( iProp = aProperties.begin() ; iProp != aProperties.end(); ++iProp )
    {
        if ( rInfo.aProperties.find( iProp->first ) != rInfo.aProperties.end() )
        {
            if ( IsPropertyRelevant( iProp->first, iProp->second ) || IsPropertyRelevant( iProp->first, rInfo.aProperties.find( iProp->first )->second ) )
            {
                if ( IsPropertyInvariant( iProp->first, iProp->second ) )
                {
                    if ( !rInfo.aProperties.find( iProp->first )->second.Equals( iProp->second ) )
                    {
                        if ( bGenErrors )
                        {
                            if ( bFixTags && IsPropertyFixable( iProp->first ) )
                            {
                                rInfo.aProperties.find( iProp->first )->second = iProp->second;
                                rInfo.SetHasBeenFixed();
{'fraction_non_alphanumeric': 0.10712641187464449, 'fraction_numerical': 0.005661041740025461, 'mean_word_length': 1.2196837611976192, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 186, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4199260', 'n_tokens_mistral': 11021, 'n_tokens_neox': 10189, 'n_words': 2225}