<commit_before>//
//  formatter.cpp
//  Parse
//
//  Created by <NAME> on 30/05/2011.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#include <sstream>

#include "formatter.h"

#include "ContextFree/grammar.h"
#include "ContextFree/item.h"
#include "ContextFree/terminal_dictionary.h"

using namespace std;
using namespace contextfree;
using namespace language;

/// \brief Turns a terminal into a string
wstring formatter::to_string(const terminal& term, const terminal_dictionary& dict) {
    // Just look up the terminal name in the dictionary
    return dict.name_for_symbol(term.symbol());
}

/// \brief Turns a nonterminal into a string
wstring formatter::to_string(const contextfree::nonterminal& nt, const contextfree::grammar& gram) {
    return gram.name_for_nonterminal(nt.symbol());
}

/// \brief Turns a guard item into a string
wstring formatter::to_string(const guard& nt, const grammar& gram, const terminal_dictionary& dict) {
    // TODO: implement me
    return L"[GUARD]";
}

/// \brief Turns an item into a string
wstring formatter::to_string(const contextfree::item& it, const grammar& gram, const terminal_dictionary& dict) {
    // Some item types can be represented quite simply
    switch (it.type()) {
        case item::empty:   return L"[empty]";
        case item::eoi:     return L"$";
        default:            break;
    }
    
    // Try to cast the item into the various different types that we know about
    const terminal* term = dynamic_cast<const terminal*>(&it);
    if (term) return to_string(*term, dict);
    
    const nonterminal* nonterm = dynamic_cast<const nonterminal*>(&it);
    if (term) return to_string(*nonterm, gram);
    
    const guard* guar = dynamic_cast<const guard*>(&it);
    if (guar) return to_string(*guar, gram, dict);
    
    // EBNF items
    wstringstream buf;
    
    const ebnf*                     eb      = dynamic_cast<const ebnf*>(&it);
    const ebnf_alternate*           alt     = dynamic_cast<const ebnf_alternate*>(&it);
    const ebnf_optional*            opt     = dynamic_cast<const ebnf_optional*>(&it);
    const ebnf_repeating*           rep     = dynamic_cast<const ebnf_repeating*>(&it);
    const ebnf_repeating_optional*  repOpt  = dynamic_cast<const ebnf_repeating_optional*>(&it);
    
    if (alt) {
        // X | Y | Z form of rule
        buf << L"(";
        
        // Convert all of the rules in this item
        bool first = true;
        for (ebnf::rule_iterator it = alt->first_rule(); it != alt->last_rule(); it++) {
            // Append the '|'
            buf << L" ";
            if (!first) buf << L"| ";
            
            // Convert the rule to a string
            buf << to_string(**it, gram, dict, -1, false);
            
            // No longer on the first item<commit_msg>Whoops, was testing the wrong thing when deciding whether or not to write out a nonterminal item<commit_after>//
//  formatter.cpp
//  Parse
//
//  Created by <NAME> on 30/05/2011.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#include <sstream>

#include "formatter.h"

#include "ContextFree/grammar.h"
#include "ContextFree/item.h"
#include "ContextFree/terminal_dictionary.h"

using namespace std;
using namespace contextfree;
using namespace language;

/// \brief Turns a terminal into a string
wstring formatter::to_string(const terminal& term, const terminal_dictionary& dict) {
    // Just look up the terminal name in the dictionary
    return dict.name_for_symbol(term.symbol());
}

/// \brief Turns a nonterminal into a string
wstring formatter::to_string(const contextfree::nonterminal& nt, const contextfree::grammar& gram) {
    return gram.name_for_nonterminal(nt.symbol());
}

/// \brief Turns a guard item into a string
wstring formatter::to_string(const guard& nt, const grammar& gram, const terminal_dictionary& dict) {
    // TODO: implement me
    return L"[GUARD]";
}

/// \brief Turns an item into a string
wstring formatter::to_string(const contextfree::item& it, const grammar& gram, const terminal_dictionary& dict) {
    // Some item types can be represented quite simply
    switch (it.type()) {
        case item::empty:   return L"[empty]";
        case item::eoi:     return L"$";
        default:            break;
    }
    
    // Try to cast the item into the various different types that we know about
    const terminal* term = dynamic_cast<const terminal*>(&it);
    if (term) return to_string(*term, dict);
    
    const nonterminal* nonterm = dynamic_cast<const nonterminal*>(&it);
    if (nonterm) return to_string(*nonterm, gram);
    
    const guard* guar = dynamic_cast<const guard*>(&it);
    if (guar) return to_string(*guar, gram, dict);
    
    // EBNF items
    wstringstream buf;
    
    const ebnf*                     eb      = dynamic_cast<const ebnf*>(&it);
    const ebnf_alternate*           alt     = dynamic_cast<const ebnf_alternate*>(&it);
    const ebnf_optional*            opt     = dynamic_cast<const ebnf_optional*>(&it);
    const ebnf_repeating*           rep     = dynamic_cast<const ebnf_repeating*>(&it);
    const ebnf_repeating_optional*  repOpt  = dynamic_cast<const ebnf_repeating_optional*>(&it);
    
    if (alt) {
        // X | Y | Z form of rule
        buf << L"(";
        
        // Convert all of the rules in this item
        bool first = true;
        for (ebnf::rule_iterator it = alt->first_rule(); it != alt->last_rule(); it++) {
            // Append the '|'
            buf << L" ";
            if (!first) buf << L"| ";
            
            // Convert the rule to a string
            buf << to_string(**it, gram, dict, -1, false);
            
            // No longer on the first item
{'fraction_non_alphanumeric': 0.11617749825296995, 'fraction_numerical': 0.004542278127183787, 'mean_word_length': 2.6534779834077855, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5357748', 'n_tokens_mistral': 1770, 'n_tokens_neox': 1706, 'n_words': 539}