<commit_before>/*=====================================================================

QGroundControl Open Source Ground Control Station

(c) 2009, 2010 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>

This file is part of the QGROUNDCONTROL project

    QGROUNDCONTROL is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    QGROUNDCONTROL is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>.

======================================================================*/

/**
 * @file
 *   @brief Implementation of class MAVLinkXMLParser
 *   @author <NAME> <<EMAIL>>
 */

#include <QFile>
#include <QDir>
#include <QPair>
#include <QList>
#include <QMap>
#include <QDateTime>
#include <QLocale>
#include "MAVLinkXMLParser.h"

#include <QDebug>

MAVLinkXMLParser::MAVLinkXMLParser(QDomDocument* document, QString outputDirectory, QObject* parent) : QObject(parent),
doc(document),
outputDirName(outputDirectory),
fileName("")
{
}

MAVLinkXMLParser::MAVLinkXMLParser(QString document, QString outputDirectory, QObject* parent) : QObject(parent)
{
    doc = new QDomDocument();
    QFile file(document);
    if (file.open(QIODevice::ReadOnly | QIODevice::Text))
    {
        const QString instanceText(QString::fromUtf8(file.readAll()));
        doc->setContent(instanceText);
    }
    fileName = document;
    outputDirName = outputDirectory;
}

MAVLinkXMLParser::~MAVLinkXMLParser()
{
}

/**
 * Generate C-code (C-89 compliant) out of the XML protocol specs.
 */
bool MAVLinkXMLParser::generate()
{
    // Process result
    bool success = true;

    // Only generate if output dir is correctly set
    if (outputDirName == "") return false;

    // print out the element names of all elements that are direct children
    // of the outermost element.
    QDomElement docElem = doc->documentElement();
    QDomNode n = docElem;//.firstChild();
    QDomNode p = docElem;

    // Sanity check variables
    QList<int>* usedMessageIDs = new QList<int>();
    QMap<QString, QString>* usedMessageNames = new QMap<QString, QString>();

    QList< QPair<QString, QString> > cFiles;
    QString lcmStructDefs = "";

    QString pureFileName;

    // Run through root children
    while(!n.isNull())
    {
        // Each child is a message
        QDomElement e = n.toElement(); // try to convert the node to an element.
        if(!e.isNull())
        {
            if (e.tagName() == "mavlink")
            {
                p = n;
                n = n.firstChild();
                while (!n.isNull())
                {
                    e = n.toElement();
                    if (!e.isNull())
                    {
                        // Handle all include tags
                        if (e.tagName() == "include")
                        {
                            QString fileName = e.text();
                            // Load file
                            QDomDocument includeDoc = QDomDocument();

                            // Prepend file path if it is a relative path and
                            // make it relative to opened file
                            QFileInfo fInfo(fileName);
                            if (fInfo.isRelative())
                            {
                                QFileInfo rInfo(this->fileName);
                                fileName = rInfo.absoluteDir().canonicalPath() + "/" + fileName;
                                pureFileName = rInfo.baseName().split(".").first();
                            }

                            QFile file(fileName);
                            if (file.open(QIODevice::ReadOnly | QIODevice::Text))
                            {
                                const QString instanceText(QString::fromUtf8(file.readAll()));
                                includeDoc.setContent(instanceText);

                                // Get all messages
                                QDomNode in = includeDoc.documentElement().firstChild();
                                //while (!in.isNull())
                                //{
                                    QDomElement ie = in.toElement();
                                    if (!ie.isNull())
                                    {
                                        if (ie.tagName() == "messages" || ie.tagName() == "include")
                                        {
                                            QDomNode ref = n.parentNode().insertAfter(in, n);
                                            if (ref.isNull())
                                            {
                                                emit parseState(QString("<font color=\"red\">ERROR: Inclusion failed: XML syntax error in file %1. Wrong/misspelled XML?\nAbort.</font>").arg(fileName));
                                                return false;
                                            }
                                        }
                                    }
                                    //in = in.nextSibling();
                                //}

                                emit parseState(QString("<font color=\"green\">Included messages from file: %1</font>").arg(fileName));
                            }
                            else
                            {
                                // Include file could not be opened
                                emit parseState(QString("<font color=\"red\">ERROR: Failed including file: %1, file is not readable. Wrong/misspelled filename?\nAbort.</font>").arg(fileName));
                                return false;
                            }

                        }
                        // Handle all message tags
                        else if (e.tagName() == "messages")
                        {
                            p = n;
                            n = n.firstChild();
                            while (!n.isNull())
                            {
                                e = n.toElement();
                                if(!e.isNull())
                                {
                                    //if (e.isNull()) continue;
                                    // Get message name
                                    QString messageName = e.attribute("name", "").toLower();
                                    if (messageName.size() == 0)
                                    {
                                        emit parseState(tr("<font color=\"red\">ERROR: Missing required name=\"\" attribute for tag %2 near line %1\nAbort.</font>").arg(QString::number(e.lineNumber()), e.tagName()));
                                        return false;
                                    }
                                    else
                                    {
                                        // Get message id
                                        bool ok;
                                        int messageId = e.attribute("id", "-1").toInt(&ok, 10);
                                        emit parseState(tr("Compiling message <strong>%1 \t(#%3)</strong> \tnear line %2").arg(messageName, QString::number(n.lineNumber()), QString::number(messageId)));

                                        // Sanity check: Accept only message IDs not used previously
                                        if (usedMessageIDs->contains(messageId))
                                        {
                                            emit parseState(tr("<font color=\"red\">ERROR: Message ID %1 used twice, second occurence near line %2 of file %3\nAbort.</font>").arg(QString::number(messageId), QString::number(e.lineNumber()), fileName));
                                            return false;
                                        }
                                        else
                                        {
                                            usedMessageIDs->append(messageId);
                                        }

                                        // Sanity check: Accept only message names not used previously
                                        if (usedMessageNames->contains(messageName))
                                        {
                                            emit parseState(tr("<font color=\"red\">ERROR: Message name %1 used twice, second occurence near line %2 of file %3\nAbort.</font>").arg(messageName, QString::number(e.lineNumber()), fileName));
                                            return false;
                                        }
                                        else
                                        {
                                            usedMessageNames->insert(messageName, QString::number(e.lineNumber()));
                                        }

                                        QString channelType = "mavlink_channel_t";
                                        QString messageType = "mavlink_message_t";

                                        // Build up function call
                                        QString commentContainer = "/**\n * @brief Send a %1 message\n *\n%2 * @return length of the message in bytes (excluding serial stream start sign)\n */\n";
                                        QString commentEntry = " * @param %1 %2\n";
                                        QString idDefine = QString("#define MAVLINK_MSG_ID_%1 %2").arg(messageName.toUpper(), QString::number(messageId));
                                        QString arrayDefines = "";
                                        QString cStructName = QString("mavlink_%1_t").arg(messageName);
                                        QString cStruct = "typedef struct __%1 \n{\n%2\n} %1;";
                                        QString cStructLines = "";
                                        QString encode = "static inline uint16_t mavlink_msg_%1_encode(uint8_t system_id, uint8_t component_id, mavlink_message_t* msg, const %2* %1)\n{\n\treturn mavlink_msg_%1_pack(%3);\n}\n";

                                        QString decode = "static inline void mavlink_msg_%1_decode(const mavlink_message_t* msg, %2* %1)\n{\n%3}\n";
                                        QString pack = "static inline uint16_t mavlink_msg_%1_pack(uint8_t system_id, uint8_t component_id, mavlink_message_t* msg%2)\n{\n\tuint16_t i = 0;\n\tmsg->msgid = MAVLINK_MSG_ID_%3;\n\n%4\n\treturn mavlink_finalize_message(msg, system_id, component_id, i);\n}\n\n";
                                        QString compactSend = "#ifdef MAVLINK_USE_CONVENIENCE_FUNCTIONS\n\nstatic inline void mavlink_msg_%3_send(%1 chan%5)\n{\n\t%2 msg;\n\tmavlink_msg_%3_pack(mavlink_system.sysid, mavlink_system.compid, &msg%4);\n\tmavlink_send_uart(chan, &msg);\n}\n\n#endif";
                                        //QString compactStructSend = "#ifdef MAVLINK_USE_CONVENIENCE_FUNCTIONS\n\nstatic inline void mavlink_msg_%3_struct_send(%1 chan%5)\n{\n\t%2 msg;\n\tmavlink_msg_%3_encode(mavlink_system.sysid, mavlink_system.compid, &msg%4);\n\tmavlink_send_uart(chan, &msg);\n}\n\n#endif";
                                        QString unpacking = "";
                                        QString prepends = "";
                                        QString packParameters = "";
                                        QString packArguments = "system_id, component_id, msg";
                                        QString packLines = "";
                                        QString decodeLines = "";
                                        QString sendArguments = "";
                                        QString commentLines = "";

                                        // Get the message fields
                                        QDomNode f = e.firstChild();
                                        while (!f.isNull())
                                        {
                                            QDomElement e2 = f.toElement();
                                            if (!e2.isNull() && e2.tagName() == "field")
                                            {
                                                QString fieldType = e2.attribute("type", "");
                                                QString fieldName = e2.attribute("name", "");
                                                QString fieldText = e2.text();

                                                // Send arguments are the same for integral types and arrays
                                                sendArguments += ", " + fieldName;

                                                // Array handling is different from simple types
                                                if (fieldType.startsWith("array"))
                                                {
                                                    int arrayLength = QString(fieldType.split("[").at(1).split("]").first()).toInt();
                                                    QString arrayType = fieldType.split("[").first();
                                                    packParameters += QString(", const ") + QString("int8_t*") + " " + fieldName;
                                                    packArguments += ", " + messageName + "->" + fieldName;

                                                    // Add field to C structure
                                                    cStructLines += QString("\t%1 %2[%3]; ///< %4\n").arg("int8_t", fieldName, QString::number(arrayLength), fieldText);
                                                    // Add pack line to message_xx_pack function
                                                    packLines += QString("\ti += put_%1_by_index(%2, %3, i, msg->payload); //%4\n").arg(arrayType, fieldName, QString::number(arrayLength), e2.text());
                                                    // Add decode function for this type
                                                    decodeLines += QString("\tmavlink_msg_%1_get_%2(msg, %1->%2);\n").arg(messageName, fieldName);
                                                    arrayDefines += QString("#define MAVLINK_MSG_%1_FIELD_%2_LEN %3").arg(messageName.toUpper(), fieldName.toUpper(), QString::number(arrayLength));
                                                }
                                                else if (fieldType.startsWith("string"))
                                                {
                                                    int arrayLength = QString(fieldType.split("[").at(1).split("]").first()).toInt();
                                                    QString arrayType = fieldType.split("[").first();
                                                    packParameters += QString(", const ") + QString("char*") + " " + fieldName;
                                                    packArguments += ", " + messageName + "->" + fieldName;

                                                    // Add field to C structure
                                                    cStructLines += QString("\t%1 %2[%3]; ///< %4\n").arg("char", fieldName, QString::number(arrayLength), fieldText);
                                                    // Add pack line to message_xx_pack function
                                                    packLines += QString("\ti += put_%1_by_index(%2, %3, i, msg->payload); //%4\n").arg(arrayType, fieldName, QString::number(arrayLength), e2.text());
                                                    // Add decode function for this type
                                                    decodeLines += QString("\tmavlink_msg_%1_get_%2(msg, %1->%2);\n").arg(messageName, fieldName);
                                                    arrayDefines += QString("#define MAVLINK_MSG_%1_FIELD_%2_LEN %3").arg(messageName.toUpper(), fieldName.toUpper(), QString::number(arrayLength));
                                                }
                                                else
                                                    // Handle simple types like integers and floats
                                                {
                                                    packParameters += ", " + fieldType + " " + fieldName;
                                                    packArguments += ", " + messageName + "->" + fieldName;

                                                    // Add field to C structure
                                                    cStructLines += QString("\t%1 %2; ///< %3\n").arg(fieldType, fieldName, fieldText);<commit_msg>Fixed minor newline issue in protocol generation<commit_after>/*=====================================================================

QGroundControl Open Source Ground Control Station

(c) 2009, 2010 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>

This file is part of the QGROUNDCONTROL project

    QGROUNDCONTROL is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    QGROUNDCONTROL is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>.

======================================================================*/

/**
 * @file
 *   @brief Implementation of class MAVLinkXMLParser
 *   @author <NAME> <<EMAIL>>
 */

#include <QFile>
#include <QDir>
#include <QPair>
#include <QList>
#include <QMap>
#include <QDateTime>
#include <QLocale>
#include "MAVLinkXMLParser.h"

#include <QDebug>

MAVLinkXMLParser::MAVLinkXMLParser(QDomDocument* document, QString outputDirectory, QObject* parent) : QObject(parent),
doc(document),
outputDirName(outputDirectory),
fileName("")
{
}

MAVLinkXMLParser::MAVLinkXMLParser(QString document, QString outputDirectory, QObject* parent) : QObject(parent)
{
    doc = new QDomDocument();
    QFile file(document);
    if (file.open(QIODevice::ReadOnly | QIODevice::Text))
    {
        const QString instanceText(QString::fromUtf8(file.readAll()));
        doc->setContent(instanceText);
    }
    fileName = document;
    outputDirName = outputDirectory;
}

MAVLinkXMLParser::~MAVLinkXMLParser()
{
}

/**
 * Generate C-code (C-89 compliant) out of the XML protocol specs.
 */
bool MAVLinkXMLParser::generate()
{
    // Process result
    bool success = true;

    // Only generate if output dir is correctly set
    if (outputDirName == "") return false;

    // print out the element names of all elements that are direct children
    // of the outermost element.
    QDomElement docElem = doc->documentElement();
    QDomNode n = docElem;//.firstChild();
    QDomNode p = docElem;

    // Sanity check variables
    QList<int>* usedMessageIDs = new QList<int>();
    QMap<QString, QString>* usedMessageNames = new QMap<QString, QString>();

    QList< QPair<QString, QString> > cFiles;
    QString lcmStructDefs = "";

    QString pureFileName;

    // Run through root children
    while(!n.isNull())
    {
        // Each child is a message
        QDomElement e = n.toElement(); // try to convert the node to an element.
        if(!e.isNull())
        {
            if (e.tagName() == "mavlink")
            {
                p = n;
                n = n.firstChild();
                while (!n.isNull())
                {
                    e = n.toElement();
                    if (!e.isNull())
                    {
                        // Handle all include tags
                        if (e.tagName() == "include")
                        {
                            QString fileName = e.text();
                            // Load file
                            QDomDocument includeDoc = QDomDocument();

                            // Prepend file path if it is a relative path and
                            // make it relative to opened file
                            QFileInfo fInfo(fileName);
                            if (fInfo.isRelative())
                            {
                                QFileInfo rInfo(this->fileName);
                                fileName = rInfo.absoluteDir().canonicalPath() + "/" + fileName;
                                pureFileName = rInfo.baseName().split(".").first();
                            }

                            QFile file(fileName);
                            if (file.open(QIODevice::ReadOnly | QIODevice::Text))
                            {
                                const QString instanceText(QString::fromUtf8(file.readAll()));
                                includeDoc.setContent(instanceText);

                                // Get all messages
                                QDomNode in = includeDoc.documentElement().firstChild();
                                //while (!in.isNull())
                                //{
                                    QDomElement ie = in.toElement();
                                    if (!ie.isNull())
                                    {
                                        if (ie.tagName() == "messages" || ie.tagName() == "include")
                                        {
                                            QDomNode ref = n.parentNode().insertAfter(in, n);
                                            if (ref.isNull())
                                            {
                                                emit parseState(QString("<font color=\"red\">ERROR: Inclusion failed: XML syntax error in file %1. Wrong/misspelled XML?\nAbort.</font>").arg(fileName));
                                                return false;
                                            }
                                        }
                                    }
                                    //in = in.nextSibling();
                                //}

                                emit parseState(QString("<font color=\"green\">Included messages from file: %1</font>").arg(fileName));
                            }
                            else
                            {
                                // Include file could not be opened
                                emit parseState(QString("<font color=\"red\">ERROR: Failed including file: %1, file is not readable. Wrong/misspelled filename?\nAbort.</font>").arg(fileName));
                                return false;
                            }

                        }
                        // Handle all message tags
                        else if (e.tagName() == "messages")
                        {
                            p = n;
                            n = n.firstChild();
                            while (!n.isNull())
                            {
                                e = n.toElement();
                                if(!e.isNull())
                                {
                                    //if (e.isNull()) continue;
                                    // Get message name
                                    QString messageName = e.attribute("name", "").toLower();
                                    if (messageName.size() == 0)
                                    {
                                        emit parseState(tr("<font color=\"red\">ERROR: Missing required name=\"\" attribute for tag %2 near line %1\nAbort.</font>").arg(QString::number(e.lineNumber()), e.tagName()));
                                        return false;
                                    }
                                    else
                                    {
                                        // Get message id
                                        bool ok;
                                        int messageId = e.attribute("id", "-1").toInt(&ok, 10);
                                        emit parseState(tr("Compiling message <strong>%1 \t(#%3)</strong> \tnear line %2").arg(messageName, QString::number(n.lineNumber()), QString::number(messageId)));

                                        // Sanity check: Accept only message IDs not used previously
                                        if (usedMessageIDs->contains(messageId))
                                        {
                                            emit parseState(tr("<font color=\"red\">ERROR: Message ID %1 used twice, second occurence near line %2 of file %3\nAbort.</font>").arg(QString::number(messageId), QString::number(e.lineNumber()), fileName));
                                            return false;
                                        }
                                        else
                                        {
                                            usedMessageIDs->append(messageId);
                                        }

                                        // Sanity check: Accept only message names not used previously
                                        if (usedMessageNames->contains(messageName))
                                        {
                                            emit parseState(tr("<font color=\"red\">ERROR: Message name %1 used twice, second occurence near line %2 of file %3\nAbort.</font>").arg(messageName, QString::number(e.lineNumber()), fileName));
                                            return false;
                                        }
                                        else
                                        {
                                            usedMessageNames->insert(messageName, QString::number(e.lineNumber()));
                                        }

                                        QString channelType = "mavlink_channel_t";
                                        QString messageType = "mavlink_message_t";

                                        // Build up function call
                                        QString commentContainer = "/**\n * @brief Send a %1 message\n *\n%2 * @return length of the message in bytes (excluding serial stream start sign)\n */\n";
                                        QString commentEntry = " * @param %1 %2\n";
                                        QString idDefine = QString("#define MAVLINK_MSG_ID_%1 %2").arg(messageName.toUpper(), QString::number(messageId));
                                        QString arrayDefines = "";
                                        QString cStructName = QString("mavlink_%1_t").arg(messageName);
                                        QString cStruct = "typedef struct __%1 \n{\n%2\n} %1;";
                                        QString cStructLines = "";
                                        QString encode = "static inline uint16_t mavlink_msg_%1_encode(uint8_t system_id, uint8_t component_id, mavlink_message_t* msg, const %2* %1)\n{\n\treturn mavlink_msg_%1_pack(%3);\n}\n";

                                        QString decode = "static inline void mavlink_msg_%1_decode(const mavlink_message_t* msg, %2* %1)\n{\n%3}\n";
                                        QString pack = "static inline uint16_t mavlink_msg_%1_pack(uint8_t system_id, uint8_t component_id, mavlink_message_t* msg%2)\n{\n\tuint16_t i = 0;\n\tmsg->msgid = MAVLINK_MSG_ID_%3;\n\n%4\n\treturn mavlink_finalize_message(msg, system_id, component_id, i);\n}\n\n";
                                        QString compactSend = "#ifdef MAVLINK_USE_CONVENIENCE_FUNCTIONS\n\nstatic inline void mavlink_msg_%3_send(%1 chan%5)\n{\n\t%2 msg;\n\tmavlink_msg_%3_pack(mavlink_system.sysid, mavlink_system.compid, &msg%4);\n\tmavlink_send_uart(chan, &msg);\n}\n\n#endif";
                                        //QString compactStructSend = "#ifdef MAVLINK_USE_CONVENIENCE_FUNCTIONS\n\nstatic inline void mavlink_msg_%3_struct_send(%1 chan%5)\n{\n\t%2 msg;\n\tmavlink_msg_%3_encode(mavlink_system.sysid, mavlink_system.compid, &msg%4);\n\tmavlink_send_uart(chan, &msg);\n}\n\n#endif";
                                        QString unpacking = "";
                                        QString prepends = "";
                                        QString packParameters = "";
                                        QString packArguments = "system_id, component_id, msg";
                                        QString packLines = "";
                                        QString decodeLines = "";
                                        QString sendArguments = "";
                                        QString commentLines = "";

                                        // Get the message fields
                                        QDomNode f = e.firstChild();
                                        while (!f.isNull())
                                        {
                                            QDomElement e2 = f.toElement();
                                            if (!e2.isNull() && e2.tagName() == "field")
                                            {
                                                QString fieldType = e2.attribute("type", "");
                                                QString fieldName = e2.attribute("name", "");
                                                QString fieldText = e2.text();

                                                // Send arguments are the same for integral types and arrays
                                                sendArguments += ", " + fieldName;

                                                // Array handling is different from simple types
                                                if (fieldType.startsWith("array"))
                                                {
                                                    int arrayLength = QString(fieldType.split("[").at(1).split("]").first()).toInt();
                                                    QString arrayType = fieldType.split("[").first();
                                                    packParameters += QString(", const ") + QString("int8_t*") + " " + fieldName;
                                                    packArguments += ", " + messageName + "->" + fieldName;

                                                    // Add field to C structure
                                                    cStructLines += QString("\t%1 %2[%3]; ///< %4\n").arg("int8_t", fieldName, QString::number(arrayLength), fieldText);
                                                    // Add pack line to message_xx_pack function
                                                    packLines += QString("\ti += put_%1_by_index(%2, %3, i, msg->payload); //%4\n").arg(arrayType, fieldName, QString::number(arrayLength), e2.text());
                                                    // Add decode function for this type
                                                    decodeLines += QString("\tmavlink_msg_%1_get_%2(msg, %1->%2);\n").arg(messageName, fieldName);
                                                    arrayDefines += QString("#define MAVLINK_MSG_%1_FIELD_%2_LEN %3\n").arg(messageName.toUpper(), fieldName.toUpper(), QString::number(arrayLength));
                                                }
                                                else if (fieldType.startsWith("string"))
                                                {
                                                    int arrayLength = QString(fieldType.split("[").at(1).split("]").first()).toInt();
                                                    QString arrayType = fieldType.split("[").first();
                                                    packParameters += QString(", const ") + QString("char*") + " " + fieldName;
                                                    packArguments += ", " + messageName + "->" + fieldName;

                                                    // Add field to C structure
                                                    cStructLines += QString("\t%1 %2[%3]; ///< %4\n").arg("char", fieldName, QString::number(arrayLength), fieldText);
                                                    // Add pack line to message_xx_pack function
                                                    packLines += QString("\ti += put_%1_by_index(%2, %3, i, msg->payload); //%4\n").arg(arrayType, fieldName, QString::number(arrayLength), e2.text());
                                                    // Add decode function for this type
                                                    decodeLines += QString("\tmavlink_msg_%1_get_%2(msg, %1->%2);\n").arg(messageName, fieldName);
                                                    arrayDefines += QString("#define MAVLINK_MSG_%1_FIELD_%2_LEN %3\n").arg(messageName.toUpper(), fieldName.toUpper(), QString::number(arrayLength));
                                                }
                                                else
                                                    // Handle simple types like integers and floats
                                                {
                                                    packParameters += ", " + fieldType + " " + fieldName;
                                                    packArguments += ", " + messageName + "->" + fieldName;

                                                    // Add field to C structure
                                                    cStructLines += QString("\t%1 %2; ///< %3\n").arg(fieldType, fieldName, fieldText);
{'fraction_non_alphanumeric': 0.10685324511841682, 'fraction_numerical': 0.007143066249011973, 'mean_word_length': 1.048575712143928, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 382, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7031388', 'n_tokens_mistral': 7160, 'n_tokens_neox': 6953, 'n_words': 1757}