<commit_before>#include "clay.hpp"
#include <cstdio>
#include <cstdlib>
#include <cstdarg>

bool shouldPrintFullMatchErrors;



//
// invoke stack - a compilation call stack
//

static vector<CompileContextEntry> contextStack;

static const unsigned RECURSION_WARNING_LEVEL = 1000;

void pushCompileContext(ObjectPtr obj) {
    if (contextStack.size() >= RECURSION_WARNING_LEVEL)
        warning("potential runaway recursion");
    if (!contextStack.empty())
        contextStack.back().location = topLocation();
    contextStack.push_back(CompileContextEntry(obj));
}

void pushCompileContext(ObjectPtr obj, const vector<ObjectPtr> &params) {
    if (contextStack.size() >= RECURSION_WARNING_LEVEL)
        warning("potential runaway recursion");
    if (!contextStack.empty())
        contextStack.back().location = topLocation();
    contextStack.push_back(CompileContextEntry(obj, params));
}

void pushCompileContext(ObjectPtr obj, const vector<ObjectPtr> &params, const vector<unsigned> &dispatchIndices) {
    if (contextStack.size() >= RECURSION_WARNING_LEVEL)
        warning("potential runaway recursion");
    if (!contextStack.empty())
        contextStack.back().location = topLocation();
    contextStack.push_back(CompileContextEntry(obj, params, dispatchIndices));
}

void popCompileContext() {
    contextStack.pop_back();
}

vector<CompileContextEntry> getCompileContext() {
    return contextStack;
}

void setCompileContext(const vector<CompileContextEntry> &x) {
    contextStack = x;
}




//
// source location of the current item being processed
//

static vector<LocationPtr> errorLocations;

void pushLocation(LocationPtr location) {
    errorLocations.push_back(location);
}

void popLocation() {
    errorLocations.pop_back();
}

LocationPtr topLocation() {
    vector<LocationPtr>::iterator i, begin;
    i = errorLocations.end();
    begin = errorLocations.begin();
    while (i != begin) {
        --i;
        if (i->ptr()) return *i;
    }
    return NULL;
}




//
// DebugPrinter
//

static vector<ObjectPtr> debugStack;

int DebugPrinter::indent = 0;

DebugPrinter::DebugPrinter(ObjectPtr obj)
    : obj(obj)
{
    for (int i = 0; i < indent; ++i)
        std::cout << ' ';
    std::cout << "BEGIN - " << obj << '\n';
    ++indent;
    debugStack.push_back(obj);
}

DebugPrinter::~DebugPrinter()
{
    debugStack.pop_back();
    --indent;
    for (int i = 0; i < indent; ++i)
        std::cout << ' ';
    std::cout << "DONE - " << obj << '\n';
}




//
// report error
//

void computeLineCol(LocationPtr location,
                    int &line,
                    int &column,
                    int &tabColumn) {
    char *p = location->source->data;
    char *end = p + location->offset;
    line = column = tabColumn = 0;
    for (; p != end; ++p) {
        ++column;
        ++tabColumn;
        if (*p == '\n') {
            ++line;
            column = 0;
            tabColumn = 0;
        }
        else if (*p == '\t') {
            tabColumn += 7;
        }
    }
}

static void splitLines(SourcePtr source, vector<string> &lines) {
    lines.push_back(string());
    char *p = source->data;
    char *end = p + source->size;
    for (; p != end; ++p) {
        lines.back().push_back(*p);
        if (*p == '\n')
            lines.push_back(string());
    }
}

static bool endsWithNewline(const string& s) {
    if (s.size() == 0) return false;
    return s[s.size()-1] == '\n';
}

static void displayLocation(LocationPtr location, int &line, int &column) {
    int tabColumn;
    computeLineCol(location, line, column, tabColumn);
    vector<string> lines;
    splitLines(location->source, lines);
    fprintf(stderr, "###############################\n");
    for (int i = line-2; i <= line+2; ++i) {
        if ((i < 0) || (i >= (int)lines.size()))
            continue;
        fprintf(stderr, "%s", lines[i].c_str());
        if (!endsWithNewline(lines[i]))
            fprintf(stderr, "\n");
        if (i == line) {
            for (int j = 0; j < tabColumn; ++j)
                fprintf(stderr, "-");
            fprintf(stderr, "^\n");
        }
    }
    fprintf(stderr, "###############################\n");
}

static void displayCompileContext() {
    if (contextStack.empty())
        return;
    fprintf(stderr, "\n");
    fprintf(stderr, "compilation context: \n");
    for (unsigned i = contextStack.size(); i > 0; --i) {
        ObjectPtr obj = contextStack[i-1].callable;
        const vector<ObjectPtr> &params = contextStack[i-1].params;

        if (i >= 1 && contextStack[i-1].location != NULL) {<commit_msg>hide redundant location info for top context frame<commit_after>#include "clay.hpp"
#include <cstdio>
#include <cstdlib>
#include <cstdarg>

bool shouldPrintFullMatchErrors;



//
// invoke stack - a compilation call stack
//

static vector<CompileContextEntry> contextStack;

static const unsigned RECURSION_WARNING_LEVEL = 1000;

void pushCompileContext(ObjectPtr obj) {
    if (contextStack.size() >= RECURSION_WARNING_LEVEL)
        warning("potential runaway recursion");
    if (!contextStack.empty())
        contextStack.back().location = topLocation();
    contextStack.push_back(CompileContextEntry(obj));
}

void pushCompileContext(ObjectPtr obj, const vector<ObjectPtr> &params) {
    if (contextStack.size() >= RECURSION_WARNING_LEVEL)
        warning("potential runaway recursion");
    if (!contextStack.empty())
        contextStack.back().location = topLocation();
    contextStack.push_back(CompileContextEntry(obj, params));
}

void pushCompileContext(ObjectPtr obj, const vector<ObjectPtr> &params, const vector<unsigned> &dispatchIndices) {
    if (contextStack.size() >= RECURSION_WARNING_LEVEL)
        warning("potential runaway recursion");
    if (!contextStack.empty())
        contextStack.back().location = topLocation();
    contextStack.push_back(CompileContextEntry(obj, params, dispatchIndices));
}

void popCompileContext() {
    contextStack.pop_back();
}

vector<CompileContextEntry> getCompileContext() {
    return contextStack;
}

void setCompileContext(const vector<CompileContextEntry> &x) {
    contextStack = x;
}




//
// source location of the current item being processed
//

static vector<LocationPtr> errorLocations;

void pushLocation(LocationPtr location) {
    errorLocations.push_back(location);
}

void popLocation() {
    errorLocations.pop_back();
}

LocationPtr topLocation() {
    vector<LocationPtr>::iterator i, begin;
    i = errorLocations.end();
    begin = errorLocations.begin();
    while (i != begin) {
        --i;
        if (i->ptr()) return *i;
    }
    return NULL;
}




//
// DebugPrinter
//

static vector<ObjectPtr> debugStack;

int DebugPrinter::indent = 0;

DebugPrinter::DebugPrinter(ObjectPtr obj)
    : obj(obj)
{
    for (int i = 0; i < indent; ++i)
        std::cout << ' ';
    std::cout << "BEGIN - " << obj << '\n';
    ++indent;
    debugStack.push_back(obj);
}

DebugPrinter::~DebugPrinter()
{
    debugStack.pop_back();
    --indent;
    for (int i = 0; i < indent; ++i)
        std::cout << ' ';
    std::cout << "DONE - " << obj << '\n';
}




//
// report error
//

void computeLineCol(LocationPtr location,
                    int &line,
                    int &column,
                    int &tabColumn) {
    char *p = location->source->data;
    char *end = p + location->offset;
    line = column = tabColumn = 0;
    for (; p != end; ++p) {
        ++column;
        ++tabColumn;
        if (*p == '\n') {
            ++line;
            column = 0;
            tabColumn = 0;
        }
        else if (*p == '\t') {
            tabColumn += 7;
        }
    }
}

static void splitLines(SourcePtr source, vector<string> &lines) {
    lines.push_back(string());
    char *p = source->data;
    char *end = p + source->size;
    for (; p != end; ++p) {
        lines.back().push_back(*p);
        if (*p == '\n')
            lines.push_back(string());
    }
}

static bool endsWithNewline(const string& s) {
    if (s.size() == 0) return false;
    return s[s.size()-1] == '\n';
}

static void displayLocation(LocationPtr location, int &line, int &column) {
    int tabColumn;
    computeLineCol(location, line, column, tabColumn);
    vector<string> lines;
    splitLines(location->source, lines);
    fprintf(stderr, "###############################\n");
    for (int i = line-2; i <= line+2; ++i) {
        if ((i < 0) || (i >= (int)lines.size()))
            continue;
        fprintf(stderr, "%s", lines[i].c_str());
        if (!endsWithNewline(lines[i]))
            fprintf(stderr, "\n");
        if (i == line) {
            for (int j = 0; j < tabColumn; ++j)
                fprintf(stderr, "-");
            fprintf(stderr, "^\n");
        }
    }
    fprintf(stderr, "###############################\n");
}

static void displayCompileContext() {
    if (contextStack.empty())
        return;
    fprintf(stderr, "\n");
    fprintf(stderr, "compilation context: \n");
    for (unsigned i = contextStack.size(); i > 0; --i) {
        ObjectPtr obj = contextStack[i-1].callable;
        const vector<ObjectPtr> &params = contextStack[i-1].params;

        if (i < contextStack.size() && contextStack[i-1].location != NULL) {
{'fraction_non_alphanumeric': 0.16366398958107228, 'fraction_numerical': 0.004666811373996093, 'mean_word_length': 2.8125775755068267, 'pattern_counts': {'":': 0, '<': 76, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1034480', 'n_tokens_mistral': 2917, 'n_tokens_neox': 2833, 'n_words': 669}