<commit_before>#include <iostream>
#include <cstdlib>
#include <unistd.h>
#include <getopt.h>
#include "fieldpropertiesdb.h"
#include "streamanalyzer.h"
#include "analyzerconfiguration.h"
using namespace std;
using namespace Strigi;

void
printDot(ostream& out, const char* locale) {
    const map<string, FieldProperties>& p
        = FieldPropertiesDb::db().allProperties();
    map<string, FieldProperties>::const_iterator i;
    out << "digraph{graph[rankdir=LR];" << endl;
    for (i = p.begin(); i != p.end(); ++i) {
        const vector<string>& parents = i->second.parentUris();
        vector<string>::const_iterator j;
        out << '"' << i->second.uri() << "\" [shape=record, label =\"" << i->second.uri() << "|{type: " << i->second.typeUri() << "}\"];" << endl;
        for (j = parents.begin(); j != parents.end(); ++j) {
            out << '"' << *j << "\"->\"" << i->second.uri() << '"' << endl;
        }
    }
    out << "}" << endl;
}
void
printRdfsProperties(ostream& out, const FieldProperties& p) {
    out << " <rdf:Property rdf:about='" << p.uri() << "'>\n"
        << "  <rdfs:label>" << p.name() << "</rdfs:label>\n"
        << "  <rdfs:comment>" << p.description() << "</rdfs:comment>\n";
    const vector<string>& parents = p.parentUris();
    vector<string>::const_iterator j;
    for (j = parents.begin(); j != parents.end(); ++j){
        out << "  <rdfs:subPropertyOf rdf:resource='" << *j << "'/>\n";
    }
    out << "  <rdfs:range rdf:resource='" << p.typeUri() << "'/>\n";
    const map<string, FieldProperties::Localized>& l = p.localized();
    map<string, FieldProperties::Localized>::const_iterator k;
    for (k = l.begin(); k != l.end(); ++k) {
        out << "  <rdfs:label xml:lang='" << k->first << "'>" << k->second.name
            << "</rdfs:label>\n";
        out << "  <rdfs:comment xml:lang='" << k->first << "'>"
            << k->second.description << "</rdfs:comment>\n";
    }
    out << "  <rdfs:domain rdf:resource='&strigi;file'/>\n"
        << " </rdf:Property>\n";
}

void<commit_msg>Added hierarchy based X.y: X becomes a new node<commit_after>#include <iostream>
#include <cstdlib>
#include <list>
#include <algorithm>
#include <unistd.h>
#include <getopt.h>
#include "fieldpropertiesdb.h"
#include "streamanalyzer.h"
#include "analyzerconfiguration.h"
using namespace std;
using namespace Strigi;

void
printDot(ostream& out, const char* locale) {
    const map<string, FieldProperties>& p
        = FieldPropertiesDb::db().allProperties();
    map<string, FieldProperties>::const_iterator i;
    // list<string> categories;
    out << "digraph{graph[rankdir=LR];" << endl;
    for (i = p.begin(); i != p.end(); ++i) {
        const vector<string>& parents = i->second.parentUris();
        vector<string>::const_iterator j;
        out << '"' << i->second.uri() << "\" [shape=record, label =\"" << i->second.uri() << "|{type: " << i->second.typeUri() << "}\"];" << endl;
        for (j = parents.begin(); j != parents.end(); ++j) {
            out << '"' << *j << "\"->\"" << i->second.uri() << '"' << endl;
        }
        // make link to category, e.g. chemistry for chemistry.inchi
        string category = i->second.uri().substr(0,i->second.uri().find("."));
        if (category.length() != i->second.uri().length()) {
            // list<string>::const_iterator match = categories.find(categories.begin(), categories.end(), category);
            // if (match != categories.end()) {
            //     categories.push_back(category);
            //     out << "\"" << category << "\" [style=filles,color=yellow]" << endl;
            //}
            out << "\"" << category << "\"->\"" << i->second.uri() << '"' << endl;
        }
    }
    out << "}" << endl;
}
void
printRdfsProperties(ostream& out, const FieldProperties& p) {
    out << " <rdf:Property rdf:about='" << p.uri() << "'>\n"
        << "  <rdfs:label>" << p.name() << "</rdfs:label>\n"
        << "  <rdfs:comment>" << p.description() << "</rdfs:comment>\n";
    const vector<string>& parents = p.parentUris();
    vector<string>::const_iterator j;
    for (j = parents.begin(); j != parents.end(); ++j){
        out << "  <rdfs:subPropertyOf rdf:resource='" << *j << "'/>\n";
    }
    out << "  <rdfs:range rdf:resource='" << p.typeUri() << "'/>\n";
    const map<string, FieldProperties::Localized>& l = p.localized();
    map<string, FieldProperties::Localized>::const_iterator k;
    for (k = l.begin(); k != l.end(); ++k) {
        out << "  <rdfs:label xml:lang='" << k->first << "'>" << k->second.name
            << "</rdfs:label>\n";
        out << "  <rdfs:comment xml:lang='" << k->first << "'>"
            << k->second.description << "</rdfs:comment>\n";
    }
    out << "  <rdfs:domain rdf:resource='&strigi;file'/>\n"
        << " </rdf:Property>\n";
}

void
{'fraction_non_alphanumeric': 0.23530633437175494, 'fraction_numerical': 0.00020768431983385254, 'mean_word_length': 2.9443079443079445, 'pattern_counts': {'":': 0, '<': 257, '<?xml version=': 0, '>': 82, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1979071', 'n_tokens_mistral': 1620, 'n_tokens_neox': 1617, 'n_words': 322}