<commit_before>#ifndef PREDCOLLECTOR_H
#define PREDCOLLECTOR_H

#include "syntax/sentencevisitor.h"
#include "syntax/atom.h"

class PredCollector : public SentenceVisitor {
public:
    PredCollector() {};
    virtual ~PredCollector() {};
    virtual void accept(const Sentence& s) {
        // only care about atoms
        const Atom* a = dynamic_cast<const Atom*>(&s);
        if (a != NULL) {
            preds.insert(*a);
        }
    }

    std::set<Atom, atomcmp> preds;
};

#endif<commit_msg>little bit faster code for predcollector (although it should be called atomcollector<commit_after>#ifndef PREDCOLLECTOR_H
#define PREDCOLLECTOR_H

#include <boost/unordered_set.hpp>
#include "syntax/sentencevisitor.h"
#include "syntax/atom.h"

class PredCollector : public SentenceVisitor {
public:
    PredCollector() {};
    virtual ~PredCollector() {};
    virtual void accept(const Sentence& s) {
        // only care about predicates
        // TODO: this should collect types, not atoms
        if (s.getTypeCode() == Atom::TypeCode) {
            preds.insert(static_cast<const Atom>(s));
        }
    }

    boost::unordered_set<Atom> preds;
};

#endif
{'fraction_non_alphanumeric': 0.11886304909560723, 'fraction_numerical': 0.0, 'mean_word_length': 3.319702602230483, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7118035', 'n_tokens_mistral': 383, 'n_tokens_neox': 370, 'n_words': 90}