<commit_before>//
//  item.cpp
//  Parse
//
//  Created by <NAME> on 30/04/2011.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#include "item.h"
#include "Lr/lr_item.h"

using namespace lr;
using namespace contextfree;


item* empty_item_constructor::construct() {
    return new empty_item();
}

void empty_item_constructor::destruct(item* i) {
    delete i;
}

/// \brief Destructor
item::~item() {
}

/// \brief Compares this item to another. Returns true if they are the same
bool item::operator==(const item& compareTo) const {
    return compareTo.type() == type() && compareTo.symbol() == symbol();
}

/// \brief Orders this item relative to another item
bool item::operator<(const item& compareTo) const {
    // Compare types
    int ourType     = type();
    int theirType   = compareTo.type();
    if (ourType < theirType) return true;
    if (theirType < ourType) return false;
    
    // Compare symbols
    return symbol() < compareTo.symbol();
}

/// \brief Comparison function, returns true if a is less than b
bool item::compare(const item& a, const item& b) {
    // Compare the kinds
    kind aKind = a.type();
    kind bKind = b.type();
    
    if (aKind < bKind) return true;
    if (aKind > bKind) return false;
    
    // For objects with an 'other' type, use RTTI to compare the underlying types
    if (aKind >= other) {
        // Compare the types of these items
        bool isBefore = typeid(a).before(typeid(b));
        
        // a is less than b if its type is before b
        if (isBefore) return true;
        
        // a is greater than b if the types aren't equal
        if (typeid(a) != typeid(b)) return false;
        
        // Call through to the items own less-than operator
        return a < b;
    }
    
    // For EBNF items, use the comparison operator built into the item
    if (aKind >= ebnf) {
        return a < b;
    }
    
    // For well-known kinds, we can just compare the symbols
    int aSym = a.symbol();
    int bSym = b.symbol();
    
    return aSym < bSym;
}

static const empty_item the_empty_item;
static item_container   an_empty_item_c((item*) &the_empty_item, false);

/// \brief Fills in the items that follow this one
void item::fill_follow(item_set& follow, const lr::lr1_item& item, const grammar& gram) const {
    // Work out what follows in the item
    const rule& rule        = *item.rule();
    int         offset      = item.offset();
    size_t      numItems    = rule.items().size();
    
    if (offset+1 >= numItems) {
        // The follow set is just the lookahead for this item
        follow = item.lookahead();
    } else {
        // The follow set is FIRST(following item)
        int followOffset = offset + 1;
        follow = gram.first(rule.items()[followOffset]);
        
        // Add further following items if the follow set can be empty, until we reach the end
        followOffset++;
        while (followOffset < numItems && follow.find(an_empty_item_c) != follow.end()) {
            // Remove the empty item
            follow.erase(an_empty_item_c);
            
            // Add the items from the next item in the rule
            const item_set& newItems = gram.first(rule.items()[followOffset]);
            follow.insert(newItems.begin(), newItems.end());
            
            // Move on to the next item
            followOffset++;
        }
        
        // If the empty set is still included, remove it and add the item lookahead
        // (Note that if the loop above terminated early, then the empty set can't be in the follow set at this point)
        if (followOffset >= numItems && follow.find(an_empty_item_c) != follow.end()) {
            follow.erase(an_empty_item_c);
            follow.insert(item.lookahead().begin(), item.lookahead().end());
        }
    }    
}

/// \brief Adds a new item to the specified set. Returns true if the set was changed
///
/// If the item is the same as an existing item except for the lookahead, the lookaheads are merged and the
/// set is the same size.
bool item::add(lr1_item_set& state, const grammar& gram, const lr1_item_container& newItem) {
    // Create a variant of the item with no lookahead
    typedef lr1_item::lookahead_set lookahead_set;
    static lookahead_set emptyLookaheadSet;
    
    lr1_item_container noLookahead(new lr1_item(&gram, newItem->rule(), newItem->offset(), emptyLookaheadSet), true);
    
    // Search for the first item that is >= noLookahead (items with any lookahead will be greater than this item)
    lr1_item_set::iterator maybeExisting = state.lower_bound(noLookahead);
    
    // If there's no rule then we can just add the item
    if (maybeExisting == state.end()) {
        state.insert(newItem);
        return true;
    }
    
    // If the rule is not the same (using the rule identifier for performance) then we can just add the item
    if ((*maybeExisting)->offset() != newItem->offset()
        || (*maybeExisting)->rule()->identifier(gram) != newItem->rule()->identifier(gram)) {
        state.insert(newItem);
        return true;
    }

    // If there are any new lookahead items, then we need to merge the lookahead sets
    // TODO: is there a quicker way of detecting when the sets are similar?
    
    // If the sets are the same, then this item has already been added and there's nothing more to do
    if (newItem->lookahead().size() == (*maybeExisting)->lookahead().size()) {
        if (newItem->lookahead() == (*maybeExisting)->lookahead()) {<commit_msg>Reset add to use the old version of itself (which creates lots of duplicates) as the new version is blatently not working at the moment.<commit_after>//
//  item.cpp
//  Parse
//
//  Created by <NAME> on 30/04/2011.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#include "item.h"
#include "Lr/lr_item.h"

using namespace lr;
using namespace contextfree;


item* empty_item_constructor::construct() {
    return new empty_item();
}

void empty_item_constructor::destruct(item* i) {
    delete i;
}

/// \brief Destructor
item::~item() {
}

/// \brief Compares this item to another. Returns true if they are the same
bool item::operator==(const item& compareTo) const {
    return compareTo.type() == type() && compareTo.symbol() == symbol();
}

/// \brief Orders this item relative to another item
bool item::operator<(const item& compareTo) const {
    // Compare types
    int ourType     = type();
    int theirType   = compareTo.type();
    if (ourType < theirType) return true;
    if (theirType < ourType) return false;
    
    // Compare symbols
    return symbol() < compareTo.symbol();
}

/// \brief Comparison function, returns true if a is less than b
bool item::compare(const item& a, const item& b) {
    // Compare the kinds
    kind aKind = a.type();
    kind bKind = b.type();
    
    if (aKind < bKind) return true;
    if (aKind > bKind) return false;
    
    // For objects with an 'other' type, use RTTI to compare the underlying types
    if (aKind >= other) {
        // Compare the types of these items
        bool isBefore = typeid(a).before(typeid(b));
        
        // a is less than b if its type is before b
        if (isBefore) return true;
        
        // a is greater than b if the types aren't equal
        if (typeid(a) != typeid(b)) return false;
        
        // Call through to the items own less-than operator
        return a < b;
    }
    
    // For EBNF items, use the comparison operator built into the item
    if (aKind >= ebnf) {
        return a < b;
    }
    
    // For well-known kinds, we can just compare the symbols
    int aSym = a.symbol();
    int bSym = b.symbol();
    
    return aSym < bSym;
}

static const empty_item the_empty_item;
static item_container   an_empty_item_c((item*) &the_empty_item, false);

/// \brief Fills in the items that follow this one
void item::fill_follow(item_set& follow, const lr::lr1_item& item, const grammar& gram) const {
    // Work out what follows in the item
    const rule& rule        = *item.rule();
    int         offset      = item.offset();
    size_t      numItems    = rule.items().size();
    
    if (offset+1 >= numItems) {
        // The follow set is just the lookahead for this item
        follow = item.lookahead();
    } else {
        // The follow set is FIRST(following item)
        int followOffset = offset + 1;
        follow = gram.first(rule.items()[followOffset]);
        
        // Add further following items if the follow set can be empty, until we reach the end
        followOffset++;
        while (followOffset < numItems && follow.find(an_empty_item_c) != follow.end()) {
            // Remove the empty item
            follow.erase(an_empty_item_c);
            
            // Add the items from the next item in the rule
            const item_set& newItems = gram.first(rule.items()[followOffset]);
            follow.insert(newItems.begin(), newItems.end());
            
            // Move on to the next item
            followOffset++;
        }
        
        // If the empty set is still included, remove it and add the item lookahead
        // (Note that if the loop above terminated early, then the empty set can't be in the follow set at this point)
        if (followOffset >= numItems && follow.find(an_empty_item_c) != follow.end()) {
            follow.erase(an_empty_item_c);
            follow.insert(item.lookahead().begin(), item.lookahead().end());
        }
    }    
}

/// \brief Adds a new item to the specified set. Returns true if the set was changed
///
/// If the item is the same as an existing item except for the lookahead, the lookaheads are merged and the
/// set is the same size.
bool item::add(lr1_item_set& state, const grammar& gram, const lr1_item_container& newItem) {
    // TODO: the stuff that follows fails to work :-/
    return state.insert(newItem).second;
    
    // Create a variant of the item with no lookahead
    typedef lr1_item::lookahead_set lookahead_set;
    static lookahead_set emptyLookaheadSet;
    
    lr1_item_container noLookahead(new lr1_item(&gram, newItem->rule(), newItem->offset(), emptyLookaheadSet), true);
    
    // Search for the first item that is >= noLookahead (items with any lookahead will be greater than this item)
    lr1_item_set::iterator maybeExisting = state.lower_bound(noLookahead);
    
    // If there's no rule then we can just add the item
    if (maybeExisting == state.end()) {
        state.insert(newItem);
        return true;
    }
    
    // If the rule is not the same (using the rule identifier for performance) then we can just add the item
    if ((*maybeExisting)->offset() != newItem->offset()
        || (*maybeExisting)->rule()->identifier(gram) != newItem->rule()->identifier(gram)) {
        state.insert(newItem);
        return true;
    }

    // If there are any new lookahead items, then we need to merge the lookahead sets
    // TODO: is there a quicker way of detecting when the sets are similar?
    
    // If the sets are the same, then this item has already been added and there's nothing more to do
    if (newItem->lookahead().size() == (*maybeExisting)->lookahead().size()) {
        if (newItem->lookahead() == (*maybeExisting)->lookahead()) {
{'fraction_non_alphanumeric': 0.10802660753880267, 'fraction_numerical': 0.0037250554323725055, 'mean_word_length': 2.4986037852932053, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6918069', 'n_tokens_mistral': 3324, 'n_tokens_neox': 3211, 'n_words': 1216}