<commit_before>/**
 * NMXmlParser.cc - uses expat lib to parse xml and load wordnet
 *
 * Copyright (c) 2001 <NAME>, <NAME>
 * All rights reserved.
 */

//#include "platform.h"

#include "NMXmlParser.h"
#include "FileXMLBufferReader.h"
#include "NMXmlDefinitions.h"
#include "SimpleTruthValue.h"
#include "Link.h" 
#include "Logger.h"

#include <math.h>
#include <stack>

extern "C" {
#include "xmlparse.h"
}

// NOTE: Any time information (including Timestamps) are represented using Atoms as follows: 
// AtTimeLink(TimeNode:<temporalStringValue>, Atom1 [, Atom2 [... , AtomN]])

Util::hash_map<char *, Handle, Util::hash<char *>, Util::eqstr> NMXmlParser::hypHandles;
bool NMXmlParser::fresh = true;
bool NMXmlParser::freshLinks = false;

//const TruthValue& NMXmlParser::DEFAULT_TV() = SimpleTruthValue(0.000001f, 0.5f);
const TruthValue& NMXmlParser::DEFAULT_TV() {
    static TruthValue* instance = new SimpleTruthValue(0.000001f, 0.5f);
    return *instance;
}

// const int TYPE_LENGTH = (int) log10((double)NUMBER_OF_CLASSES) + 1;

typedef struct {
    unsigned enabled: 1;
    unsigned ignoring: 1;
    unsigned processNodes: 1;
    unsigned processRelationships: 1;
} Status;

typedef std::stack<void*> ParserStack;
// TODO: CREATE AN ABSTRACT CLASS "Stackable" for check for subclasses 
// when poping elements from stack.

typedef struct {
    AtomSpace* atomSpace;
    //AtomTable* atomTable;
    ParserStack stack;
    Handle lastInsertedHandle;
    Status status;
} UserData;

void push(ParserStack& ps, void* p) {
    ps.push(p);
}

void* top(ParserStack& ps) {
    if (!ps.size()) return NULL;
    return ps.top();
}

void* pop(ParserStack& ps) {
    if (!ps.size()) return NULL;
    void* ret = ps.top();
    ps.pop();  
    return ret;
}

Type getTypeForString(const char *name, bool onlyClassName)
{
    if (!name) return 0;

    Type result = ClassServer::getType(name);
    if (result != NOTYPE) {          
        return result;
    }

    if (strcmp(name, POLYGON_CORNER_TOKEN) != 0 && 
        (onlyClassName || strcmp(name, ELEMENT_TOKEN) != 0))
    {
      MAIN_LOGGER.log(Util::Logger::ERROR, "Warning: null type for name returned! (%s)\n", name);
    }
  
    return 0;
}

char *nativeBuildLinkKey(Atom *link){
    char key[1<<16];
    char aux[1 << 16];

    key[0] = '\0';

    sprintf(aux, "%d ",link->getType());
    strcat(key, aux);

    for (int i = 0; i < link->getArity(); i++){
        sprintf(aux, "%p ", TLB::getHandle(link->getOutgoingAtom(i)));
        strcat(key, aux);
    }

    return strdup(key);
}

//#include <sys/time.h>
//unsigned long int cumulativeParseNodesStart = 0;
//unsigned long int cumulativeParseLinksStart = 0;
//unsigned long int cumulativeParseEnd = 0;
//unsigned long int cumulativeParseEnd1 = 0;
//unsigned long int cumulativeParseEnd2 = 0;

static void nativeStartElement(void *userData, const char *name, const char **atts)
    throw (RuntimeException, InconsistenceException)
{
    Atom* r = NULL;
    Type typeFound;
    
    UserData* ud = (UserData*) userData;

    // precesses head tags (list, tagdescription, etc)
    if (strcmp(name, LIST_TOKEN) == 0) {
        ud->status.enabled = 1;
        return;
    } else if (strcmp(name, TAG_DESCRIPTION_TOKEN) == 0) {
        ud->status.ignoring = 1;
        return;
    }

    // abandons element if necessary
    if ((!ud->status.enabled) || ud->status.ignoring)
        return;

    typeFound = getTypeForString(name, false);

    // processes nodes
    if (ud->status.processNodes) {
//        timeval s;
//        gettimeofday(&s, NULL);

<commit_msg>mark a bunch of internal use functions as static<commit_after>/**
 * NMXmlParser.cc - uses expat lib to parse xml and load wordnet
 *
 * Copyright (c) 2001 <NAME>, <NAME>
 * All rights reserved.
 */

//#include "platform.h"

#include "NMXmlParser.h"
#include "FileXMLBufferReader.h"
#include "NMXmlDefinitions.h"
#include "SimpleTruthValue.h"
#include "Link.h" 
#include "Logger.h"

#include <math.h>
#include <stack>

extern "C" {
#include "xmlparse.h"
}

// NOTE: Any time information (including Timestamps) are represented using Atoms as follows: 
// AtTimeLink(TimeNode:<temporalStringValue>, Atom1 [, Atom2 [... , AtomN]])

Util::hash_map<char *, Handle, Util::hash<char *>, Util::eqstr> NMXmlParser::hypHandles;
bool NMXmlParser::fresh = true;
bool NMXmlParser::freshLinks = false;

//const TruthValue& NMXmlParser::DEFAULT_TV() = SimpleTruthValue(0.000001f, 0.5f);
const TruthValue& NMXmlParser::DEFAULT_TV() {
    static TruthValue* instance = new SimpleTruthValue(0.000001f, 0.5f);
    return *instance;
}

// const int TYPE_LENGTH = (int) log10((double)NUMBER_OF_CLASSES) + 1;

typedef struct {
    unsigned enabled: 1;
    unsigned ignoring: 1;
    unsigned processNodes: 1;
    unsigned processRelationships: 1;
} Status;

typedef std::stack<void*> ParserStack;
// TODO: CREATE AN ABSTRACT CLASS "Stackable" for check for subclasses 
// when poping elements from stack.

typedef struct {
    AtomSpace* atomSpace;
    //AtomTable* atomTable;
    ParserStack stack;
    Handle lastInsertedHandle;
    Status status;
} UserData;

static void push(ParserStack& ps, void* p) {
    ps.push(p);
}

static void* top(ParserStack& ps) {
    if (!ps.size()) return NULL;
    return ps.top();
}

static void* pop(ParserStack& ps) {
    if (!ps.size()) return NULL;
    void* ret = ps.top();
    ps.pop();  
    return ret;
}

static Type getTypeForString(const char *name, bool onlyClassName)
{
    if (!name) return 0;

    Type result = ClassServer::getType(name);
    if (result != NOTYPE) {          
        return result;
    }

    if (strcmp(name, POLYGON_CORNER_TOKEN) != 0 && 
        (onlyClassName || strcmp(name, ELEMENT_TOKEN) != 0))
    {
      MAIN_LOGGER.log(Util::Logger::ERROR, "Warning: null type for name returned! (%s)\n", name);
    }
  
    return 0;
}

static char *nativeBuildLinkKey(Atom *link)
{
    char key[1<<16];
    char aux[1 << 16];

    key[0] = '\0';

    sprintf(aux, "%d ",link->getType());
    strcat(key, aux);

    for (int i = 0; i < link->getArity(); i++){
        sprintf(aux, "%p ", TLB::getHandle(link->getOutgoingAtom(i)));
        strcat(key, aux);
    }

    return strdup(key);
}

//#include <sys/time.h>
//unsigned long int cumulativeParseNodesStart = 0;
//unsigned long int cumulativeParseLinksStart = 0;
//unsigned long int cumulativeParseEnd = 0;
//unsigned long int cumulativeParseEnd1 = 0;
//unsigned long int cumulativeParseEnd2 = 0;

static void nativeStartElement(void *userData, const char *name, const char **atts)
    throw (RuntimeException, InconsistenceException)
{
    Atom* r = NULL;
    Type typeFound;
    
    UserData* ud = (UserData*) userData;

    // processes head tags (list, tagdescription, etc)
    if (strcmp(name, LIST_TOKEN) == 0) {
        ud->status.enabled = 1;
        return;
    } else if (strcmp(name, TAG_DESCRIPTION_TOKEN) == 0) {
        ud->status.ignoring = 1;
        return;
    }

    // abandons element if necessary
    if ((!ud->status.enabled) || ud->status.ignoring)
        return;

    typeFound = getTypeForString(name, false);

    // processes nodes
    if (ud->status.processNodes) {
//        timeval s;
//        gettimeofday(&s, NULL);


{'fraction_non_alphanumeric': 0.14086066140860662, 'fraction_numerical': 0.015220700152207, 'mean_word_length': 3.3621001810500903, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5685870', 'n_tokens_mistral': 2544, 'n_tokens_neox': 2404, 'n_words': 651}