<commit_before>#include "clang/AST/ASTConsumer.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/FrontendAction.h"
#include "clang/Tooling/CommonOptionsParser.h"
#include "clang/Tooling/Tooling.h"
#include "clang/Lex/Preprocessor.h"

#include <iostream>
#include <sstream>

using namespace clang;
using namespace clang::tooling;

std::string escapeString(std::string const& input)
{
    std::ostringstream ss;
    for (auto c : input)
    {
        switch (c)
        {
        case '\\':
            ss << "\\\\";
            break;
        case '"':
            ss << "\\\"";
            break;
        case '/':
            ss << "\\/";
            break;
        case '\b':
            ss << "\\b";
            break;
        case '\f':
            ss << "\\f";
            break;
        case '\n':
            ss << "\\n";
            break;
        case '\r':
            ss << "\\r";
            break;
        case '\t':
            ss << "\\t";
            break;
        default:
            ss << c;
            break;
        }
    }
    return ss.str();
}

class DumpVisitor : public RecursiveASTVisitor<DumpVisitor>
{
  public:
    explicit DumpVisitor(ASTContext* context) : context(context) {}

    void TraverseNewScope(Stmt* stmt)
    {
        if (!isa<CompoundStmt>(stmt))
        {
            depth++;
            WriteDepth();
        }

        TraverseStmt(stmt);

        if (!isa<CompoundStmt>(stmt))
        {
            std::cout << "\n";
            depth--;
        }
    }

    bool TraverseStmt(Stmt* stmt)
    {
        if (!stmt)
            return RecursiveASTVisitor::TraverseStmt(stmt);

        if (auto compoundStmt = dyn_cast<CompoundStmt>(stmt))
        {
            depth++;
            for (auto stmt : compoundStmt->body())
            {
                if (!stmt)
                    continue;

                WriteDepth();
                TraverseStmt(stmt);
                std::cout << "\n";
            }
            depth--;
            return true;
        }

        if (auto callExpr = dyn_cast<CallExpr>(stmt))
        {
            TraverseStmt(callExpr->getCallee());
            bool first = true;
            std::cout << "(";
            for (auto param : callExpr->arguments())
            {
                if (!first)
                    std::cout << ", ";

                TraverseStmt(param);
                first = false;
            }
            std::cout << ")";
            return true;
        }

        if (auto ifStmt = dyn_cast<IfStmt>(stmt))
        {
            std::cout << "if ";
            TraverseStmt(ifStmt->getCond());
            std::cout << " then\n";

            TraverseNewScope(ifStmt->getThen());

            WriteDepth();
            std::cout << "end";
            return true;
        }

        if (auto arraySubscriptExpr = dyn_cast<ArraySubscriptExpr>(stmt))
        {
            TraverseStmt(arraySubscriptExpr->getBase());
            std::cout << "[";
            TraverseStmt(arraySubscriptExpr->getIdx());
            // Add 1 to compensate for 1-based indexing
            std::cout << "+1]";
            return true;
        }

        if (auto binaryOperator = dyn_cast<BinaryOperator>(stmt))
        {
            // TODO: Replace this with something more robust
            TraverseStmt(binaryOperator->getLHS());
            auto opcodeStr = binaryOperator->getOpcodeStr().str();

            if (binaryOperator->isCompoundAssignmentOp())
            {
                opcodeStr = opcodeStr.substr(0, opcodeStr.find_last_of('='));
                std::cout << " = ";
                TraverseStmt(binaryOperator->getLHS());
            }

            std::cout << " " << opcodeStr << " ";
            TraverseStmt(binaryOperator->getRHS());
            return true;
        }

        if (auto parenExpr = dyn_cast<ParenExpr>(stmt))
        {
            std::cout << "(";
            TraverseStmt(parenExpr->getSubExpr());
            std::cout << ")";<commit_msg>Added parens to indexing to fix precedence issues<commit_after>#include "clang/AST/ASTConsumer.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/FrontendAction.h"
#include "clang/Tooling/CommonOptionsParser.h"
#include "clang/Tooling/Tooling.h"
#include "clang/Lex/Preprocessor.h"

#include <iostream>
#include <sstream>

using namespace clang;
using namespace clang::tooling;

std::string escapeString(std::string const& input)
{
    std::ostringstream ss;
    for (auto c : input)
    {
        switch (c)
        {
        case '\\':
            ss << "\\\\";
            break;
        case '"':
            ss << "\\\"";
            break;
        case '/':
            ss << "\\/";
            break;
        case '\b':
            ss << "\\b";
            break;
        case '\f':
            ss << "\\f";
            break;
        case '\n':
            ss << "\\n";
            break;
        case '\r':
            ss << "\\r";
            break;
        case '\t':
            ss << "\\t";
            break;
        default:
            ss << c;
            break;
        }
    }
    return ss.str();
}

class DumpVisitor : public RecursiveASTVisitor<DumpVisitor>
{
  public:
    explicit DumpVisitor(ASTContext* context) : context(context) {}

    void TraverseNewScope(Stmt* stmt)
    {
        if (!isa<CompoundStmt>(stmt))
        {
            depth++;
            WriteDepth();
        }

        TraverseStmt(stmt);

        if (!isa<CompoundStmt>(stmt))
        {
            std::cout << "\n";
            depth--;
        }
    }

    bool TraverseStmt(Stmt* stmt)
    {
        if (!stmt)
            return RecursiveASTVisitor::TraverseStmt(stmt);

        if (auto compoundStmt = dyn_cast<CompoundStmt>(stmt))
        {
            depth++;
            for (auto stmt : compoundStmt->body())
            {
                if (!stmt)
                    continue;

                WriteDepth();
                TraverseStmt(stmt);
                std::cout << "\n";
            }
            depth--;
            return true;
        }

        if (auto callExpr = dyn_cast<CallExpr>(stmt))
        {
            TraverseStmt(callExpr->getCallee());
            bool first = true;
            std::cout << "(";
            for (auto param : callExpr->arguments())
            {
                if (!first)
                    std::cout << ", ";

                TraverseStmt(param);
                first = false;
            }
            std::cout << ")";
            return true;
        }

        if (auto ifStmt = dyn_cast<IfStmt>(stmt))
        {
            std::cout << "if ";
            TraverseStmt(ifStmt->getCond());
            std::cout << " then\n";

            TraverseNewScope(ifStmt->getThen());

            WriteDepth();
            std::cout << "end";
            return true;
        }

        if (auto arraySubscriptExpr = dyn_cast<ArraySubscriptExpr>(stmt))
        {
            TraverseStmt(arraySubscriptExpr->getBase());
            std::cout << "[(";
            TraverseStmt(arraySubscriptExpr->getIdx());
            // Add 1 to compensate for 1-based indexing
            std::cout << ")+1]";
            return true;
        }

        if (auto binaryOperator = dyn_cast<BinaryOperator>(stmt))
        {
            // TODO: Replace this with something more robust
            TraverseStmt(binaryOperator->getLHS());
            auto opcodeStr = binaryOperator->getOpcodeStr().str();

            if (binaryOperator->isCompoundAssignmentOp())
            {
                opcodeStr = opcodeStr.substr(0, opcodeStr.find_last_of('='));
                std::cout << " = ";
                TraverseStmt(binaryOperator->getLHS());
            }

            std::cout << " " << opcodeStr << " ";
            TraverseStmt(binaryOperator->getRHS());
            return true;
        }

        if (auto parenExpr = dyn_cast<ParenExpr>(stmt))
        {
            std::cout << "(";
            TraverseStmt(parenExpr->getSubExpr());
            std::cout << ")";
{'fraction_non_alphanumeric': 0.13941480206540446, 'fraction_numerical': 0.0009835259404966806, 'mean_word_length': 1.3982900943396226, 'pattern_counts': {'":': 0, '<': 125, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1604721', 'n_tokens_mistral': 2315, 'n_tokens_neox': 2284, 'n_words': 433}