<commit_before>/**
 * NMXmlParser.cc - uses expat lib to parse xml and load wordnet
 *
 * Copyright (c) 2001 <NAME>, <NAME>
 * All rights reserved.
 *
 */

#include "NMXmlParser.h"
#include "FileXMLBufferReader.h"
#include "NMXmlDefinitions.h"
#include "SimpleTruthValue.h"
#include "Link.h" 
#include "Logger.h"

#include <math.h>
#include <stack>
#include <expat.h>

/*
 * XXX To be fixed: remove all of the uses of "throw" in this code, 
 * to be replaced by a gentler error mechanism. The problem is two-fold:
 * the "throws" are made from within routines that are called by the
 * external libxmlparser library, which is written in C, not C++. 
 * Thus, these errors can't ever actually be caught, since they never
 * get through the parser. As a result, any minor mistake in an XML
 * will cause the entire server to crash, which is just plain ugly.
 * Yuck!

 * NOTE: Any time information (including Timestamps) are represented using Atoms as follows: 
 * AtTimeLink(TimeNode:<temporalStringValue>, Atom1 [, Atom2 [... , AtomN]])
 */

Util::hash_map<char *, Handle, Util::hash<char *>, Util::eqstr> NMXmlParser::hypHandles;
bool NMXmlParser::fresh = true;
bool NMXmlParser::freshLinks = false;

//const TruthValue& NMXmlParser::DEFAULT_TV() = SimpleTruthValue(0.000001f, 0.5f);
const TruthValue& NMXmlParser::DEFAULT_TV() {
    static TruthValue* instance = new SimpleTruthValue(0.000001f, 0.5f);
    return *instance;
}

// const int TYPE_LENGTH = (int) log10((double)NUMBER_OF_CLASSES) + 1;

typedef struct {
    unsigned enabled: 1;
    unsigned ignoring: 1;
    unsigned processNodes: 1;
    unsigned processRelationships: 1;
} Status;

typedef std::stack<void*> ParserStack;
// TODO: CREATE AN ABSTRACT CLASS "Stackable" for check for subclasses 
// when poping elements from stack.

typedef struct {
    AtomSpace* atomSpace;
    //AtomTable* atomTable;
    ParserStack stack;
    Handle lastInsertedHandle;
    Status status;
} UserData;

static void push(ParserStack& ps, void* p) {
    ps.push(p);
}

static void* top(ParserStack& ps) {
    if (!ps.size()) return NULL;
    return ps.top();<commit_msg>change the tdefault truth values to be "true", not false, and set the default confidence to 0.5, if not otherwise specified.<commit_after>/**
 * NMXmlParser.cc - uses expat lib to parse xml and load wordnet
 *
 * Copyright (c) 2001 <NAME>, <NAME>
 * All rights reserved.
 *
 */

#include "NMXmlParser.h"
#include "FileXMLBufferReader.h"
#include "NMXmlDefinitions.h"
#include "SimpleTruthValue.h"
#include "Link.h" 
#include "Logger.h"

#include <math.h>
#include <stack>
#include <expat.h>

/*
 * XXX To be fixed: remove all of the uses of "throw" in this code, 
 * to be replaced by a gentler error mechanism. The problem is two-fold:
 * the "throws" are made from within routines that are called by the
 * external libxmlparser library, which is written in C, not C++. 
 * Thus, these errors can't ever actually be caught, since they never
 * get through the parser. As a result, any minor mistake in an XML
 * will cause the entire server to crash, which is just plain ugly.
 * Yuck!

 * NOTE: Any time information (including Timestamps) are represented using Atoms as follows: 
 * AtTimeLink(TimeNode:<temporalStringValue>, Atom1 [, Atom2 [... , AtomN]])
 */

Util::hash_map<char *, Handle, Util::hash<char *>, Util::eqstr> NMXmlParser::hypHandles;
bool NMXmlParser::fresh = true;
bool NMXmlParser::freshLinks = false;

// If unspecified, the default truth value is "true", with confidence of 0.5
const TruthValue& NMXmlParser::DEFAULT_TV()
{
    static SimpleTruthValue* instance = NULL;
    if (instance == NULL)
    {
        instance = new SimpleTruthValue(1.0f, 1.0f);
        instance->setConfidence(0.5);
    }
    return *instance;
}

// const int TYPE_LENGTH = (int) log10((double)NUMBER_OF_CLASSES) + 1;

typedef struct {
    unsigned enabled: 1;
    unsigned ignoring: 1;
    unsigned processNodes: 1;
    unsigned processRelationships: 1;
} Status;

typedef std::stack<void*> ParserStack;
// TODO: CREATE AN ABSTRACT CLASS "Stackable" for check for subclasses 
// when poping elements from stack.

typedef struct {
    AtomSpace* atomSpace;
    //AtomTable* atomTable;
    ParserStack stack;
    Handle lastInsertedHandle;
    Status status;
} UserData;

static void push(ParserStack& ps, void* p) {
    ps.push(p);
}

static void* top(ParserStack& ps) {
    if (!ps.size()) return NULL;
    return ps.top();
{'fraction_non_alphanumeric': 0.11343686698176908, 'fraction_numerical': 0.012153950033760972, 'mean_word_length': 4.0442678774120315, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7357815', 'n_tokens_mistral': 1474, 'n_tokens_neox': 1381, 'n_words': 500}