<commit_before>/*
 * opencog/xml/NMXmlParser.cc
 *
 * Copyright (C) 2002-2007 Novamente LLC
 * All Rights Reserved
 *
 * Written by <NAME> <<EMAIL>>
 *            <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <stack>
#include <string>

#include <math.h>
#include <expat.h>

#include <opencog/util/platform.h>

#include <opencog/atomspace/Link.h>
#include <opencog/atomspace/SimpleTruthValue.h>
#include <opencog/util/Logger.h>
#include <opencog/xml/FileXMLBufferReader.h>
#include <opencog/xml/NMXmlDefinitions.h>
#include <opencog/xml/NMXmlParser.h>

/*
 * XXX To be fixed: remove all of the uses of "throw" in this code,
 * to be replaced by a gentler error mechanism. The problem is two-fold:
 * the "throws" are made from within routines that are called by the
 * external libxmlparser library, which is written in C, not C++.
 * Thus, these errors can't ever actually be caught, since they never
 * get through the parser. As a result, any minor mistake in an XML
 * will cause the entire server to crash, which is just plain ugly.
 * Yuck!

 * NOTE: Any time information (including Timestamps) are represented using Atoms as follows:
 * AtTimeLink(TimeNode:<temporalStringValue>, Atom1 [, Atom2 [... , AtomN]])
 */

using namespace opencog;

std::tr1::unordered_map<const std::string, Handle, hash<std::string>, std::equal_to<std::string> > NMXmlParser::hypHandles;
bool NMXmlParser::fresh = true;
bool NMXmlParser::freshLinks = false;

// If unspecified, the default truth value is "true", with confidence of 0.5
const TruthValue& NMXmlParser::DEFAULT_TV()
{
    static SimpleTruthValue* instance = NULL;
    if (instance == NULL) {
        instance = new SimpleTruthValue(1.0f, 1.0f);
        instance->setConfidence(0.5);
    }
    return *instance;
}

// const int TYPE_LENGTH = (int) log10((double)NUMBER_OF_CLASSES) + 1;

typedef struct {
unsigned enabled: 1;
unsigned ignoring: 1;
unsigned processNodes: 1;
unsigned processRelationships: 1;
} Status;

typedef std::stack<void*> ParserStack;
// TODO: CREATE AN ABSTRACT CLASS "Stackable" for check for subclasses
// when poping elements from stack.

typedef struct {
    AtomSpace* atomSpace;
    //AtomTable* atomTable;
    ParserStack stack;
    Handle lastInsertedHandle;
    Status status;
    XML_Parser parser;
} UserData;

static void push(ParserStack& ps, void* p)
{
    ps.push(p);
}

static void* top(ParserStack& ps)
{
    if (!ps.size()) return NULL;
    return ps.top();
}

static void* pop(ParserStack& ps)
{
    if (!ps.size()) return NULL;
    void* ret = ps.top();
    ps.pop();
    return ret;
}

/**
 * Return the type corresponding to the type name
 */
static Type getTypeFromString(const char *name, bool onlyClassName)
{
    if (!name) return NOTYPE;

    Type result = classserver().getType(name);
    if (result != NOTYPE) {
        return result;
    }

    if (strcmp(name, POLYGON_CORNER_TOKEN) != 0 &&
            (onlyClassName || strcmp(name, ELEMENT_TOKEN) != 0)) {
        logger().error("Warning: null type for name returned! (%s)\n", name);
    }

    return 0;
}

#ifdef NOT_USED_RIGHT_NOW
static char *nativeBuildLinkKey(Atom *link)
{
    char key[1<<16];
    char aux[1 << 16];

    key[0] = '\0';

    sprintf(aux, "%d ", link->getType());
    strcat(key, aux);

    for (int i = 0; i < link->getArity(); i++) {
        sprintf(aux, "%p ", TLB::getHandle(link->getOutgoingAtom(i)));
        strcat(key, aux);
    }

    return strdup(key);
}
#endif /* NOT_USED_RIGHT_NOW */

//#include <sys/time.h>
//unsigned long int cumulativeParseNodesStart = 0;
//unsigned long int cumulativeParseLinksStart = 0;
//unsigned long int cumulativeParseEnd = 0;
//unsigned long int cumulativeParseEnd1 = 0;
//unsigned long int cumulativeParseEnd2 = 0;


/**
 * Handle attributes that are common to both Nodes and Links.
 * In other words, handle "confidence" and "strength" attributes.
 */
static const char ** scan_common_attrs (Atom *r, const char **atts)
{
    float buffer;
    if (strcmp(*atts, CONFIDENCE_TOKEN) == 0) {
        atts++;
        sscanf(*atts, "%f", &buffer);
        if (buffer == 1.0) {
            throw RuntimeException(TRACE_INFO,
                                   "fatal error: confidence can not be 1.0");
        }
        SimpleTruthValue newTv((const SimpleTruthValue&)r->getTruthValue());
        newTv.setConfidence(buffer);<commit_msg>remove inappropriate check for confidence values ... this check should be handled in other places (e.g. PLN) and not in the parser.<commit_after>/*
 * opencog/xml/NMXmlParser.cc
 *
 * Copyright (C) 2002-2007 Novamente LLC
 * All Rights Reserved
 *
 * Written by <NAME> <<EMAIL>>
 *            <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <stack>
#include <string>

#include <math.h>
#include <expat.h>

#include <opencog/util/platform.h>

#include <opencog/atomspace/Link.h>
#include <opencog/atomspace/SimpleTruthValue.h>
#include <opencog/util/Logger.h>
#include <opencog/xml/FileXMLBufferReader.h>
#include <opencog/xml/NMXmlDefinitions.h>
#include <opencog/xml/NMXmlParser.h>

/*
 * XXX To be fixed: remove all of the uses of "throw" in this code,
 * to be replaced by a gentler error mechanism. The problem is two-fold:
 * the "throws" are made from within routines that are called by the
 * external libxmlparser library, which is written in C, not C++.
 * Thus, these errors can't ever actually be caught, since they never
 * get through the parser. As a result, any minor mistake in an XML
 * will cause the entire server to crash, which is just plain ugly.
 * Yuck!

 * NOTE: Any time information (including Timestamps) are represented using Atoms as follows:
 * AtTimeLink(TimeNode:<temporalStringValue>, Atom1 [, Atom2 [... , AtomN]])
 */

using namespace opencog;

std::tr1::unordered_map<const std::string, Handle, hash<std::string>, std::equal_to<std::string> > NMXmlParser::hypHandles;
bool NMXmlParser::fresh = true;
bool NMXmlParser::freshLinks = false;

// If unspecified, the default truth value is "true", with confidence of 0.5
const TruthValue& NMXmlParser::DEFAULT_TV()
{
    static SimpleTruthValue* instance = NULL;
    if (instance == NULL) {
        instance = new SimpleTruthValue(1.0f, 1.0f);
        instance->setConfidence(0.5);
    }
    return *instance;
}

// const int TYPE_LENGTH = (int) log10((double)NUMBER_OF_CLASSES) + 1;

typedef struct {
unsigned enabled: 1;
unsigned ignoring: 1;
unsigned processNodes: 1;
unsigned processRelationships: 1;
} Status;

typedef std::stack<void*> ParserStack;
// TODO: CREATE AN ABSTRACT CLASS "Stackable" for check for subclasses
// when poping elements from stack.

typedef struct {
    AtomSpace* atomSpace;
    //AtomTable* atomTable;
    ParserStack stack;
    Handle lastInsertedHandle;
    Status status;
    XML_Parser parser;
} UserData;

static void push(ParserStack& ps, void* p)
{
    ps.push(p);
}

static void* top(ParserStack& ps)
{
    if (!ps.size()) return NULL;
    return ps.top();
}

static void* pop(ParserStack& ps)
{
    if (!ps.size()) return NULL;
    void* ret = ps.top();
    ps.pop();
    return ret;
}

/**
 * Return the type corresponding to the type name
 */
static Type getTypeFromString(const char *name, bool onlyClassName)
{
    if (!name) return NOTYPE;

    Type result = classserver().getType(name);
    if (result != NOTYPE) {
        return result;
    }

    if (strcmp(name, POLYGON_CORNER_TOKEN) != 0 &&
            (onlyClassName || strcmp(name, ELEMENT_TOKEN) != 0)) {
        logger().error("Warning: null type for name returned! (%s)\n", name);
    }

    return 0;
}

#ifdef NOT_USED_RIGHT_NOW
static char *nativeBuildLinkKey(Atom *link)
{
    char key[1<<16];
    char aux[1 << 16];

    key[0] = '\0';

    sprintf(aux, "%d ", link->getType());
    strcat(key, aux);

    for (int i = 0; i < link->getArity(); i++) {
        sprintf(aux, "%p ", TLB::getHandle(link->getOutgoingAtom(i)));
        strcat(key, aux);
    }

    return strdup(key);
}
#endif /* NOT_USED_RIGHT_NOW */

//#include <sys/time.h>
//unsigned long int cumulativeParseNodesStart = 0;
//unsigned long int cumulativeParseLinksStart = 0;
//unsigned long int cumulativeParseEnd = 0;
//unsigned long int cumulativeParseEnd1 = 0;
//unsigned long int cumulativeParseEnd2 = 0;


/**
 * Handle attributes that are common to both Nodes and Links.
 * In other words, handle "confidence" and "strength" attributes.
 */
static const char ** scan_common_attrs (Atom *r, const char **atts)
{
    float buffer;
    if (strcmp(*atts, CONFIDENCE_TOKEN) == 0) {
        atts++;
        sscanf(*atts, "%f", &buffer);
        SimpleTruthValue newTv((const SimpleTruthValue&)r->getTruthValue());
        newTv.setConfidence(buffer);
{'fraction_non_alphanumeric': 0.11887765219115606, 'fraction_numerical': 0.01189767995240928, 'mean_word_length': 3.6228230980751603, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4595304', 'n_tokens_mistral': 3271, 'n_tokens_neox': 3112, 'n_words': 1077}