<commit_before>package nl.mpi.kinnate.entityindexer;

import java.awt.BorderLayout;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.IOException;
import java.io.StringReader;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.JTextArea;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.transform.stream.StreamSource;
import nl.mpi.arbil.GuiHelper;
import nl.mpi.arbil.LinorgBugCatcher;
import nl.mpi.arbil.LinorgSessionStorage;
import nl.mpi.kinnate.svg.GraphDataNode;
import org.basex.core.BaseXException;
import org.basex.core.Context;
import org.basex.core.cmd.Set;
import org.basex.core.cmd.CreateDB;
import org.basex.core.cmd.DropDB;
import org.basex.core.cmd.XQuery;
import org.basex.query.QueryException;
import org.basex.query.QueryProcessor;
import org.basex.query.item.Item;
import org.basex.query.iter.Iter;

/**
 *  Document   : EntityCollection
 *  Created on : Feb 15, 2011, 5:37:06 PM
 *  Author     : <NAME>
 */
public class EntityCollection implements EntityService {

    private String databaseName = "nl-mpi-kinnate";
    static Context context = new Context();

    public EntityCollection() {
    }

    // todo: move this into the graphdatanode
    @XmlRootElement(name = "results")
    static public class RelationResults {

        @XmlElementWrapper(name = "relations")
        @XmlElement(name = "entity")
        RelationData[] relationArray;
    }

    static public class RelationData {

        @XmlElement
        GraphDataNode.RelationType type;
        @XmlElement
        String path;
    }

    public class SearchResults {

        public String[] resultsPathArray;
        public String statusMessage;
        public int resultCount = 0;
    }

    public void createDatabase() {
        try {
            new DropDB(databaseName).execute(context);
            new Set("CREATEFILTER", "*.cmdi").execute(context);
            new CreateDB(databaseName, LinorgSessionStorage.getSingleInstance().getCacheDirectory().toString()).execute(context);
//            context.close();
        } catch (BaseXException baseXException) {
            GuiHelper.linorgBugCatcher.logError(baseXException);
        }
    }

    public RelationData[] getRelatedNodes(String uniqueIdentifier, IndexerParameters indexParameters) {
        // todo: it would seem that entityPath is not going to be adequate because of resolved vs unresolved paths, it would seem best at this point to implement an ID or even persistent identifier if posible
        // there are two parts required to get all relations of an ego: check the ego entity for relations to others and then check the relations of all other entities for references to the ego entity
        // for now maybe use an md5 sum the full path url or something and put it into both the entity and linking entities
        QueryBuilder queryBuilder = new QueryBuilder();
        String query1String = queryBuilder.getRelationQuery(uniqueIdentifier, indexParameters);

        System.out.println("query1String: " + query1String);
        ArrayList<RelationData> resultsArray = new ArrayList<RelationData>();
        try {
            JAXBContext jaxbContext = JAXBContext.newInstance(RelationResults.class);
            Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
            String queryResult = new XQuery(query1String).execute(context);
            RelationResults relationResults = (RelationResults) unmarshaller.unmarshal(new StreamSource(new StringReader(queryResult)), RelationResults.class).getValue();
            resultsArray.addAll(Arrays.asList(relationResults.relationArray));
        } catch (JAXBException exception) {
            new LinorgBugCatcher().logError(exception);
        } catch (BaseXException exception) {
            new LinorgBugCatcher().logError(exception);
        }
        return resultsArray.toArray(new RelationData[]{});
    }

    public SearchResults listAllRelationTypes() {
        // todo: use this to populate the InderParametersFormUI
        String queryString = "distinct-values(collection('nl-mpi-kinnate')/Kinnate/Relation/Type/text())";
        return performQuery(queryString);
    }

    public SearchResults searchByName(String namePartString) {
        String queryString = "for $doc in collection('nl-mpi-kinnate') where contains(string-join($doc//text()), \"" + namePartString + "\") return base-uri($doc)";
        return performQuery(queryString);
    }

    private SearchResults performQuery(String queryString) {
        SearchResults searchResults = new SearchResults();
        ArrayList<String> resultPaths = new ArrayList<String>();
        try {
            //for $doc in collection('nl-mpi-kinnate')  where $doc//NAME="Bob /Cox/" return base-uri($doc)
//            String query = "for $doc in collection('nl-mpi-kinnate') where $doc//NAME = \"" + namePartString + "\" return base-uri($doc)";
            QueryProcessor proc = new QueryProcessor(queryString, context);//Emp[contains(Ename,"AR")]
            Iter iter = proc.iter();
            Item item;
            while ((item = iter.next()) != null) {
//                System.out.println(item.toJava());
                resultPaths.add(item.toJava().toString());
                searchResults.resultCount++;
            }
            proc.close();
<commit_msg>Added a separate class to build queries that retrive entities and their relations. Added JAXB annotations to the graph data nodes so they can be deserialized directly from the retrieved XML.<commit_after>package nl.mpi.kinnate.entityindexer;

import java.awt.BorderLayout;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.IOException;
import java.io.StringReader;
import java.net.URI;
import java.util.ArrayList;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.JTextArea;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.transform.stream.StreamSource;
import nl.mpi.arbil.GuiHelper;
import nl.mpi.arbil.LinorgBugCatcher;
import nl.mpi.arbil.LinorgSessionStorage;
import nl.mpi.kinnate.svg.GraphDataNode;
import org.basex.core.BaseXException;
import org.basex.core.Context;
import org.basex.core.cmd.Set;
import org.basex.core.cmd.CreateDB;
import org.basex.core.cmd.DropDB;
import org.basex.core.cmd.XQuery;
import org.basex.query.QueryException;
import org.basex.query.QueryProcessor;
import org.basex.query.item.Item;
import org.basex.query.iter.Iter;

/**
 *  Document   : EntityCollection
 *  Created on : Feb 15, 2011, 5:37:06 PM
 *  Author     : <NAME>
 */
public class EntityCollection implements EntityService {

    private String databaseName = "nl-mpi-kinnate";
    static Context context = new Context();

    public EntityCollection() {
    }

    public class SearchResults {

        public String[] resultsPathArray;
        public String statusMessage;
        public int resultCount = 0;
    }

    public void createDatabase() {
        try {
            new DropDB(databaseName).execute(context);
            new Set("CREATEFILTER", "*.cmdi").execute(context);
            new CreateDB(databaseName, LinorgSessionStorage.getSingleInstance().getCacheDirectory().toString()).execute(context);
//            context.close();
        } catch (BaseXException baseXException) {
            GuiHelper.linorgBugCatcher.logError(baseXException);
        }
    }

//    public RelationData[] getRelatedNodes(String uniqueIdentifier, IndexerParameters indexParameters) {
//        // todo: it would seem that entityPath is not going to be adequate because of resolved vs unresolved paths, it would seem best at this point to implement an ID or even persistent identifier if posible
//        // there are two parts required to get all relations of an ego: check the ego entity for relations to others and then check the relations of all other entities for references to the ego entity
//        // for now maybe use an md5 sum the full path url or something and put it into both the entity and linking entities
//        QueryBuilder queryBuilder = new QueryBuilder();
//        String query1String = "<results>" + queryBuilder.getRelationQuery(uniqueIdentifier, indexParameters) + "</results>";
//
//        System.out.println("query1String: " + query1String);
//        ArrayList<RelationData> resultsArray = new ArrayList<RelationData>();
//        try {
//            JAXBContext jaxbContext = JAXBContext.newInstance(RelationResults.class);
//            Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
//            String queryResult = new XQuery(query1String).execute(context);
//            RelationResults relationResults = (RelationResults) unmarshaller.unmarshal(new StreamSource(new StringReader(queryResult)), RelationResults.class).getValue();
//            resultsArray.addAll(Arrays.asList(relationResults.relationArray));
//        } catch (JAXBException exception) {
//            new LinorgBugCatcher().logError(exception);
//        } catch (BaseXException exception) {
//            new LinorgBugCatcher().logError(exception);
//        }
//        return resultsArray.toArray(new RelationData[]{});
//    }

    public SearchResults listAllRelationTypes() {
        // todo: use this to populate the InderParametersFormUI
        String queryString = "distinct-values(collection('nl-mpi-kinnate')/Kinnate/Relation/Type/text())";
        return performQuery(queryString);
    }

    public SearchResults searchByName(String namePartString) {
        String queryString = "for $doc in collection('nl-mpi-kinnate') where contains(string-join($doc//text()), \"" + namePartString + "\") return base-uri($doc)";
        return performQuery(queryString);
    }

    private SearchResults performQuery(String queryString) {
        SearchResults searchResults = new SearchResults();
        ArrayList<String> resultPaths = new ArrayList<String>();
        try {
            //for $doc in collection('nl-mpi-kinnate')  where $doc//NAME="Bob /Cox/" return base-uri($doc)
//            String query = "for $doc in collection('nl-mpi-kinnate') where $doc//NAME = \"" + namePartString + "\" return base-uri($doc)";
            QueryProcessor proc = new QueryProcessor(queryString, context);//Emp[contains(Ename,"AR")]
            Iter iter = proc.iter();
            Item item;
            while ((item = iter.next()) != null) {
//                System.out.println(item.toJava());
                resultPaths.add(item.toJava().toString());
                searchResults.resultCount++;
            }
            proc.close();

{'fraction_non_alphanumeric': 0.10612097066280334, 'fraction_numerical': 0.0030785947120608474, 'mean_word_length': 3.2826677006591702, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '374464', 'n_tokens_mistral': 3055, 'n_tokens_neox': 2979, 'n_words': 762}