<commit_before>%{
/*
    rtf.ll  -  A simple RTF Parser (Flex code)

    Copyright (c) 2002 by <NAME>   <<EMAIL>>    (original code)
    Copyright (c) 2004 by <NAME> <<EMAIL>> (Kopete port)
    Kopete    (c) 2002-2003 by the Kopete developers  <<EMAIL>>

    *************************************************************************
    *                                                                       *
    * This program is free software; you can redistribute it and/or modify  *
    * it under the terms of the GNU General Public License as published by  *
    * the Free Software Foundation; either version 2 of the License, or     *
    * (at your option) any later version.                                   *
    *                                                                       *
    *************************************************************************
*/

#define UP			1
#define DOWN			2
#define CMD			3
#define TXT			4
#define HEX			5
#define IMG			6
#define UNICODE_CHAR		7
#define SKIP			8
#define SLASH			9
#define S_TXT			10

#define YY_NEVER_INTERACTIVE	1
#define YY_ALWAYS_INTERACTIVE	0
#define YY_MAIN			0

%}

%option nounput
%option nostack
%option prefix="rtf"

%%

"{"				{ return UP; }
"}"				{ return DOWN; }
"\\"[\\\{\}]			{ return SLASH; }
"\\u"[0-9]{3,7}"?"		{ return UNICODE_CHAR; }
"\\"[A-Za-z]+[0-9]*[ ]? 	{ return CMD; }
"\\'"[0-9A-Fa-f][0-9A-Fa-f]	{ return HEX; }
"<##"[^>]+">"			{ return IMG; }
[^\\{}<]+			{ return TXT; }
.				{ return TXT; }
%%

#include "rtf2html.h"

void ParStyle::clearFormatting()
{
   // For now, do nothing.
   // dir is not a formatting item.
}

QString RTF2HTML::quoteString(const QString &_str, quoteMode mode)
{
    QString str = _str;
    str.replace(QRegExp("&"), "&amp;");
    str.replace(QRegExp("<"), "&lt;");
    str.replace(QRegExp(">"), "&gt;");
    str.replace(QRegExp("\""), "&quot;");
    str.replace(QRegExp("\r"), "");
    switch (mode){
    case quoteHTML:
        str.replace(QRegExp("\n"), "<br>\n");
        break;
    case quoteXML:
        str.replace(QRegExp("\n"), "<br/>\n");
        break;
    default:
        break;
    }
    QRegExp re("  +");
    int len;
    int pos = 0;

    while ((pos = re.search(str, pos)) != -1) {
        len = re.matchedLength();

        if (len == 1)
            continue;
        QString s = " ";
        for (int i = 1; i < len; i++)
            s += "&nbsp;";
        str.replace(pos, len, s);
    }
    return str;
}

OutTag* RTF2HTML::getTopOutTag(TagEnum tagType)
{
    vector<OutTag>::iterator it, it_end;
    for(it = oTags.begin(), it_end = oTags.end(); it != it_end; ++it)
       if (it->tag == tagType)
        return &(*it);
    return NULL;
}

void RTF2HTML::FlushOutTags()
{
    vector<OutTag>::iterator iter;
    for (iter = oTags.begin(); iter != oTags.end(); iter++)
    {
        OutTag &t = *iter;
        switch (t.tag){
        case TAG_FONT_COLOR:
            {
               // RTF colors are 1-based; colors[] is a 0-based array.
               QColor &c = colors[t.param-1];
               PrintUnquoted("<span style=\"color:#%02X%02X%02X\">", c.red(), c.green(), c.blue());
            }
            break;
        case TAG_FONT_SIZE:
            PrintUnquoted("<span style=\"font-size:%upt\">", t.param);
            break;
        case TAG_FONT_FAMILY:
            {
               FontDef &f = fonts[t.param-1];
               string name = (!f.nonTaggedName.empty()) ? f.nonTaggedName : f.taggedName;
               PrintUnquoted("<span style=\"font-family:%s\">", name.c_str());
            }
            break;
        case TAG_BG_COLOR:{
                QColor &c = colors[t.param];
                PrintUnquoted("<span style=\"background-color:#%02X%02X%02X;\">", c.red(), c.green(), c.blue());
                break;
            }
        case TAG_BOLD:
            PrintUnquoted("<b>");
            break;
        case TAG_ITALIC:
            PrintUnquoted("<i>");
            break;
        case TAG_UNDERLINE:
            PrintUnquoted("<u>");
            break;
        default:
            break;
        }
    }
    oTags.clear();
}

// This function will close the already-opened tag 'tag'. It will take
// care of closing the tags which 'tag' contains first (ie. it will unroll
// the stack till the point where 'tag' is).
void Level::resetTag(TagEnum tag)
{
    // A stack which'll keep tags we had to close in order to reach 'tag'.
    // After we close 'tag', we will reopen them.
    stack<TagEnum> s;

    while (p->tags.size() > m_nTagsStartPos){ // Don't go further than the point where this level starts.

        TagEnum nTag = p->tags.top();

        /* A tag will be located in oTags if it still wasn't printed out.
           A tag will get printed out only if necessary (e.g. <I></I> will
           be optimized away).
           Thus, for each tag we remove from the actual tag stack, we also
           try to remove a yet-to-be-printed tag, and only if there are no
           yet-to-be-printed tags left, we start closing the tags we pop.
        */
        if (p->oTags.empty()){
            switch (nTag){
            case TAG_FONT_COLOR:
            case TAG_FONT_SIZE:
            case TAG_BG_COLOR:
            case TAG_FONT_FAMILY:
                p->PrintUnquoted("</span>");
                break;
            case TAG_BOLD:
                p->PrintUnquoted("</b>");
                break;
            case TAG_ITALIC:
                p->PrintUnquoted("</i>");
                break;
            case TAG_UNDERLINE:
                p->PrintUnquoted("</u>");
                break;
            default:
                break;
            }
        }else{
            p->oTags.pop_back();
        }

        p->tags.pop();
        if (nTag == tag) break; // if we reached the tag we were looking to close.
        s.push(nTag); // remember to reopen this tag
    }

    if (tag == TAG_ALL) return;

    while (!s.empty()){
        TagEnum nTag = s.top();
        switch (nTag){
        case TAG_FONT_COLOR:{
                unsigned nFontColor = m_nFontColor;
                m_nFontColor = 0;
                setFontColor(nFontColor);
                break;
            }
        case TAG_FONT_SIZE:{
                unsigned nFontSize = m_nFontSize;
                m_nFontSize = 0;
                setFontSize(nFontSize);<commit_msg>update rtf.ll with stuff from sim cvs<commit_after>%{
/*
    rtf.ll  -  A simple RTF Parser (Flex code)

    Copyright (c) 2002 by <NAME>   <<EMAIL>>    (original code)
    Copyright (c) 2004 by <NAME> <<EMAIL>> (Kopete port)
    Kopete    (c) 2002-2003 by the Kopete developers  <<EMAIL>>

    *************************************************************************
    *                                                                       *
    * This program is free software; you can redistribute it and/or modify  *
    * it under the terms of the GNU General Public License as published by  *
    * the Free Software Foundation; either version 2 of the License, or     *
    * (at your option) any later version.                                   *
    *                                                                       *
    *************************************************************************
*/

#define UP			1
#define DOWN			2
#define CMD			3
#define TXT			4
#define HEX			5
#define IMG			6
#define UNICODE_CHAR		7
#define SKIP			8
#define SLASH			9
#define S_TXT			10

#define YY_NEVER_INTERACTIVE	1
#define YY_ALWAYS_INTERACTIVE	0
#define YY_MAIN			0

%}

%option nounput
%option nostack
%option prefix="rtf"

%%

"{"				{ return UP; }
"}"				{ return DOWN; }
"\\"[\\\{\}]			{ return SLASH; }
"\\u"[0-9]{3,7}"?"		{ return UNICODE_CHAR; }
"\\"[A-Za-z]+[0-9]*[ ]? 	{ return CMD; }
"\\'"[0-9A-Fa-f][0-9A-Fa-f]	{ return HEX; }
"<##"[^>]+">"			{ return IMG; }
[^\\{}<]+			{ return TXT; }
.				{ return TXT; }
%%

#include "rtf2html.h"

void ParStyle::clearFormatting()
{
   // For now, do nothing.
   // dir is not a formatting item.
}

QString RTF2HTML::quoteString(const QString &_str, quoteMode mode)
{
    QString str = _str;
    str.replace(QRegExp("&"), "&amp;");
    str.replace(QRegExp("<"), "&lt;");
    str.replace(QRegExp(">"), "&gt;");
    str.replace(QRegExp("\""), "&quot;");
    str.replace(QRegExp("\r"), "");
    switch (mode){
    case quoteHTML:
        str.replace(QRegExp("\n"), "<br>\n");
        break;
    case quoteXML:
        str.replace(QRegExp("\n"), "<br/>\n");
        break;
    default:
        break;
    }
    QRegExp re("  +");
    int len;
    int pos = 0;

    while ((pos = re.search(str, pos)) != -1) {
        len = re.matchedLength();

        if (len == 1)
            continue;
        QString s = " ";
        for (int i = 1; i < len; i++)
            s += "&nbsp;";
        str.replace(pos, len, s);
    }
    return str;
}

OutTag* RTF2HTML::getTopOutTag(TagEnum tagType)
{
    vector<OutTag>::iterator it, it_end;
    for(it = oTags.begin(), it_end = oTags.end(); it != it_end; ++it)
       if (it->tag == tagType)
        return &(*it);
    return NULL;
}

void RTF2HTML::FlushOutTags()
{
    vector<OutTag>::iterator iter;
    for (iter = oTags.begin(); iter != oTags.end(); iter++)
    {
        OutTag &t = *iter;
        switch (t.tag){
        case TAG_FONT_COLOR:
            {
               // RTF colors are 1-based; colors[] is a 0-based array.
               QColor &c = colors[t.param-1];
               PrintUnquoted("<span style=\"color:#%02X%02X%02X\">", c.red(), c.green(), c.blue());
            }
            break;
        case TAG_FONT_SIZE:
            PrintUnquoted("<span style=\"font-size:%upt\">", t.param);
            break;
        case TAG_FONT_FAMILY:
            {
               FontDef &f = fonts[t.param-1];
               string name = (!f.nonTaggedName.empty()) ? f.nonTaggedName : f.taggedName;
               PrintUnquoted("<span style=\"font-family:%s\">", name.c_str());
            }
            break;
        case TAG_BG_COLOR:{
                QColor &c = colors[t.param];
                PrintUnquoted("<span style=\"background-color:#%02X%02X%02X;\">", c.red(), c.green(), c.blue());
                break;
            }
        case TAG_BOLD:
            PrintUnquoted("<b>");
            break;
        case TAG_ITALIC:
            PrintUnquoted("<i>");
            break;
        case TAG_UNDERLINE:
            PrintUnquoted("<u>");
            break;
        default:
            break;
        }
    }
    oTags.clear();
}

// This function will close the already-opened tag 'tag'. It will take
// care of closing the tags which 'tag' contains first (ie. it will unroll
// the stack till the point where 'tag' is).
void Level::resetTag(TagEnum tag)
{
    // A stack which'll keep tags we had to close in order to reach 'tag'.
    // After we close 'tag', we will reopen them.
    stack<TagEnum> s;

    while (p->tags.size() > m_nTagsStartPos){ // Don't go further than the point where this level starts.

        TagEnum nTag = p->tags.top();

        /* A tag will be located in oTags if it still wasn't printed out.
           A tag will get printed out only if necessary (e.g. <I></I> will
           be optimized away).
           Thus, for each tag we remove from the actual tag stack, we also
           try to remove a yet-to-be-printed tag, and only if there are no
           yet-to-be-printed tags left, we start closing the tags we pop.
           The tags have one space - needed for umlaute () and .utf8()
        */
        if (p->oTags.empty()){
            switch (nTag){
            case TAG_FONT_COLOR:
            case TAG_FONT_SIZE:
            case TAG_BG_COLOR:
            case TAG_FONT_FAMILY:
                p->PrintUnquoted(" </span>");
                break;
            case TAG_BOLD:
                p->PrintUnquoted(" </b>");
                break;
            case TAG_ITALIC:
                p->PrintUnquoted(" </i>");
                break;
            case TAG_UNDERLINE:
                p->PrintUnquoted(" </u>");
                break;
            default:
                break;
            }
        }else{
            p->oTags.pop_back();
        }

        p->tags.pop();
        if (nTag == tag) break; // if we reached the tag we were looking to close.
        s.push(nTag); // remember to reopen this tag
    }

    if (tag == TAG_ALL) return;

    while (!s.empty()){
        TagEnum nTag = s.top();
        switch (nTag){
        case TAG_FONT_COLOR:{
                unsigned nFontColor = m_nFontColor;
                m_nFontColor = 0;
                setFontColor(nFontColor);
                break;
            }
        case TAG_FONT_SIZE:{
                unsigned nFontSize = m_nFontSize;
                m_nFontSize = 0;
                setFontSize(nFontSize);
{'fraction_non_alphanumeric': 0.16039138943248532, 'fraction_numerical': 0.010567514677103719, 'mean_word_length': 1.708501165995336, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1192625', 'n_tokens_mistral': 4272, 'n_tokens_neox': 3993, 'n_words': 1093}