<commit_before>#include <clang-c/Index.h>
#include <cstdio>

#include <iostream>
#include <vector>


struct client_data
{
    CXTranslationUnit tu;
    std::vector<std::string> namespaces;
    bool in_struct;
};

CXChildVisitResult visitor (CXCursor cursor, CXCursor parent, CXClientData data_)
{
    client_data& data = *static_cast<client_data*>(data_);

    CXCursorKind kind = clang_getCursorKind(cursor);

    std::cout
        << clang_getCString(clang_getCursorKindSpelling(clang_getCursorKind(cursor))) << " "
        << clang_getCString(clang_getCursorSpelling(cursor)) << " "
        << "\n";

    CXSourceRange range = clang_getCursorExtent(cursor);
    CXSourceLocation start = clang_getRangeStart(range);
    CXSourceLocation end = clang_getRangeEnd(range);
    unsigned int start_offset, end_offset;

    CXToken* tokens = 0;
    unsigned int num_tokens = 0;
    clang_tokenize(data.tu, range, &tokens, &num_tokens);
    for (unsigned int i = 0; i < num_tokens; ++i) {
        if (i)
            std::cout << " ";
        std::cout << clang_getCString(clang_getTokenSpelling(data.tu, tokens[i]));
    }
    std::cout << "\n";
    clang_disposeTokens(data.tu, tokens, num_tokens);

    return CXChildVisit_Recurse;
}

int main (int argc, char *argv[])
{
    CXIndex index = clang_createIndex(0, 1);
    CXTranslationUnit tu =
        clang_parseTranslationUnit(index, 0, argv, argc, 0, 0, CXTranslationUnit_None);

    client_data data = {tu, {}, false};

    CXCursor cursor = clang_getTranslationUnitCursor(tu);
    clang_visitChildren(cursor, visitor, &data);

    clang_disposeTranslationUnit(tu);
    clang_disposeIndex(index);

    return 0;
}
<commit_msg>Grooming.<commit_after>#include <clang-c/Index.h>
#include <cstdio>

#include <iostream>
#include <vector>


struct client_data
{
    CXTranslationUnit tu;
    std::vector<std::string> namespaces;
    bool in_struct;
};

CXChildVisitResult
visitor (CXCursor cursor, CXCursor parent, CXClientData data_)
{
    client_data& data = *static_cast<client_data*>(data_);

    CXCursorKind kind = clang_getCursorKind(cursor);

    CXString kind_spelling = clang_getCursorKindSpelling(kind);
    CXString cursor_spelling = clang_getCursorSpelling(cursor);
    std::cout << clang_getCString(kind_spelling) << " "
              << clang_getCString(cursor_spelling) << " "
              << "\n";

    CXSourceRange range = clang_getCursorExtent(cursor);
    CXSourceLocation start = clang_getRangeStart(range);
    CXSourceLocation end = clang_getRangeEnd(range);
    unsigned int start_offset, end_offset;

    CXToken* tokens = 0;
    unsigned int num_tokens = 0;
    clang_tokenize(data.tu, range, &tokens, &num_tokens);
    for (unsigned int i = 0; i < num_tokens; ++i) {
        if (i)
            std::cout << " ";
        std::cout
            << clang_getCString(clang_getTokenSpelling(data.tu, tokens[i]));
    }
    std::cout << "\n";
    clang_disposeTokens(data.tu, tokens, num_tokens);

    return CXChildVisit_Recurse;
}

int main (int argc, char *argv[])
{
    CXIndex index = clang_createIndex(0, 1);
    CXTranslationUnit tu = clang_parseTranslationUnit(
        index,
        0,
        argv,
        argc,
        0,
        0,
        CXTranslationUnit_None
    );

    client_data data = {tu, {}, false};

    CXCursor cursor = clang_getTranslationUnitCursor(tu);
    clang_visitChildren(cursor, visitor, &data);

    clang_disposeTranslationUnit(tu);
    clang_disposeIndex(index);

    return 0;
}

{'fraction_non_alphanumeric': 0.11678622668579627, 'fraction_numerical': 0.005164992826398852, 'mean_word_length': 3.611111111111111, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4811521', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1138, 'n_words': 231}