<commit_before>package nl.mpi.kinnate.entityindexer;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import nl.mpi.arbil.GuiHelper;
import nl.mpi.arbil.LinorgSessionStorage;
import nl.mpi.arbil.clarin.CmdiComponentBuilder;
import nl.mpi.kinnate.entityindexer.QueryParser.ParserHighlight;
import nl.mpi.kinnate.kindata.DataTypes;
import nl.mpi.kinnate.kindata.EntityData;
import nl.mpi.kinnate.kintypestrings.KinTypeStringConverter;
import nl.mpi.kinnate.kintypestrings.KinTypeStringConverter.KinType;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

/**
 *  Document   : EntityIndex
 *  Created on : Feb 2, 2011, 17:29:36 PM
 *  Author     : <NAME>
 */
public class EntityIndex implements EntityService {

//    IndexerParameters indexParameters;
    private HashMap<String, IndexerEntityData> knownEntities;
//    private EntityCollection entityCollection;

    public EntityIndex() {
//        indexParameters = indexParametersLocal;
//        entityCollection = new EntityCollection();
        knownEntities = new HashMap<String, IndexerEntityData>();
    }

    private IndexerEntityData getEntityData(String egoEntityUriString, IndexerParameters indexParameters) throws URISyntaxException {
        URI egoEntityUri = new URI(egoEntityUriString);
        return getEntityData(egoEntityUri, indexParameters);
    }

    private IndexerEntityData getEntityData(URI egoEntityUri, IndexerParameters indexParameters) {
        IndexerEntityData entityData = knownEntities.get(egoEntityUri.toASCIIString());
        if (entityData != null) {
            return entityData;
        } else {
            entityData = new IndexerEntityData(null); // todo: while this could pass the identifier it is unlikely that this class will use them as it relies on the url instead
            knownEntities.put(egoEntityUri.toASCIIString(), entityData);
            try {
                Document linksDom = new CmdiComponentBuilder().getDocument(egoEntityUri);
                NodeList relationLinkNodeList = org.apache.xpath.XPathAPI.selectNodeList(linksDom, indexParameters.linkPath);
                for (int nodeCounter = 0; nodeCounter < relationLinkNodeList.getLength(); nodeCounter++) {
                    Node relationLinkNode = relationLinkNodeList.item(nodeCounter);
                    if (relationLinkNode != null) {
                        // resolve the alter URL against its ego URI
                        URI alterUri = egoEntityUri.resolve(relationLinkNode.getTextContent());
                        entityData.addRelation(alterUri.toASCIIString());
                        // get any requested link data
                        for (String[] relevantDataPath : indexParameters.relevantLinkData.getValues()) {
                            for (Node linkDataNode = relationLinkNode.getParentNode().getFirstChild(); linkDataNode != null; linkDataNode = linkDataNode.getNextSibling()) {
                                if (relevantDataPath[0].equals(linkDataNode.getNodeName())) {
                                    entityData.addRelationData(alterUri.toASCIIString(), relevantDataPath[0], linkDataNode.getTextContent());
                                }
                            }
                        }
                    }
                }
                // get any requested entity data
                for (String[] relevantDataPath : indexParameters.getRelevantEntityData()) {
                    NodeList relevantaDataNodeList = org.apache.xpath.XPathAPI.selectNodeList(linksDom, relevantDataPath[0]);
                    for (int dataCounter = 0; dataCounter < relevantaDataNodeList.getLength(); dataCounter++) {
                        Node dataNode = relevantaDataNodeList.item(dataCounter);
                        if (dataNode != null) {
                            entityData.addEntityData(relevantDataPath[0], dataNode.getTextContent());
                        }
                    }
                }
            } catch (TransformerException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            } catch (ParserConfigurationException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            } catch (DOMException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            } catch (IOException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            } catch (SAXException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            }
            return entityData;
        }
    }

    public void printKnownEntities() {
        for (String currentEgo : knownEntities.keySet()) {
            System.out.println("currentEgo: " + currentEgo);
            IndexerEntityData currentEntityData = knownEntities.get(currentEgo);
            for (String[] currentRecord : currentEntityData.getEntityFields()) {
                System.out.println("-> entityField: " + currentRecord[0] + " : " + currentRecord[1]);
            }
            for (String currentLink : currentEntityData.getRelationPaths()) {
                System.out.println("--> currentLink: " + currentLink);
                for (String[] currentRecord : currentEntityData.getRelationData(currentLink)) {
                    System.out.println("---> linkField: " + currentRecord[0] + " : " + currentRecord[1]);
                }
            }
        }
    }

    public void loadAllEntities(IndexerParameters indexParameters) {
        String[] treeNodesArray = LinorgSessionStorage.getSingleInstance().loadStringArray("KinGraphTree");
        if (treeNodesArray != null) {
            for (String currentNodeString : treeNodesArray) {
                try {
                    URI egoEntityUri = new URI(currentNodeString);
                    getEntityData(egoEntityUri, indexParameters);
                } catch (URISyntaxException exception) {
                    GuiHelper.linorgBugCatcher.logError(exception);
                }
            }
        }
    }

    public void setKinTypeStringTerm(String symbolString, String fieldPath, String fieldValue) {
        // todo: set the terms that combine to form the kin type strings
        // eg: setKinTypeStringTerm("M", "Kinnate.Gedcom.Entity.SEX", "F");
    }

    private EntityData getGraphDataNode(boolean isEgo, URI entityUri, IndexerParameters indexParameters) {
        IndexerEntityData entityData = getEntityData(entityUri, indexParameters);
        ArrayList<String> labelTextList = new ArrayList<String>();
        for (String[] currentLabelField : indexParameters.labelFields.getValues()) {
            String labelTextTemp = entityData.getEntityField(currentLabelField[0]);
            if (labelTextTemp != null) {
                labelTextList.add(labelTextTemp);
            }
        }
        for (String currentSymbolField[] : indexParameters.symbolFieldsFields.getValues()) {
            String linkSymbolString = entityData.getEntityField(currentSymbolField[0]);
            if (linkSymbolString != null) {
                return new EntityData(entityData.getUniqueIdentifier(), entityUri.toASCIIString(), null, currentSymbolField[1], labelTextList.toArray(new String[]{}), isEgo);
            }
        }
        return new EntityData(entityData.getUniqueIdentifier(), entityUri.toASCIIString(), null, EntityData.SymbolType.none, labelTextList.toArray(new String[]{}), isEgo);
    }

    private void setRelationData(EntityData egoNode, EntityData alterNode, IndexerEntityData egoData, String alterPath, IndexerParameters indexParameters) {
        DataTypes.RelationType egoType = null;
        DataTypes.RelationType alterType = null;
        String[][] alterRelationFields = egoData.getRelationData(alterPath);
        if (alterRelationFields != null) {
            for (String[] ancestorField : indexParameters.ancestorFields.getValues()) {
                for (String[] egoRelationField : alterRelationFields) {
                    if (ancestorField[0].equals(egoRelationField[1])) {
                        egoType = DataTypes.RelationType.ancestor;
                        alterType = DataTypes.RelationType.descendant;
                    }
                }
            }
            for (String[] ancestorField : indexParameters.decendantFields.getValues()) {
                for (String[] egoRelationField : alterRelationFields) {
                    if (ancestorField[0].equals(egoRelationField[1])) {<commit_msg>Added the use of jaxb to save the current settings used to generate the graph in the graph svg file. Added the use of jaxb to load indexersettings and kin parameters from the svg. Separated Index Parameters into a separate class.<commit_after>package nl.mpi.kinnate.entityindexer;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import nl.mpi.arbil.GuiHelper;
import nl.mpi.arbil.LinorgSessionStorage;
import nl.mpi.arbil.clarin.CmdiComponentBuilder;
import nl.mpi.kinnate.entityindexer.QueryParser.ParserHighlight;
import nl.mpi.kinnate.kindata.DataTypes;
import nl.mpi.kinnate.kindata.EntityData;
import nl.mpi.kinnate.kintypestrings.KinTypeStringConverter;
import nl.mpi.kinnate.kintypestrings.KinTypeStringConverter.KinType;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

/**
 *  Document   : EntityIndex
 *  Created on : Feb 2, 2011, 17:29:36 PM
 *  Author     : <NAME>
 */
public class EntityIndex implements EntityService {

//    IndexerParameters indexParameters;
    private HashMap<String, IndexerEntityData> knownEntities;
//    private EntityCollection entityCollection;

    public EntityIndex() {
//        indexParameters = indexParametersLocal;
//        entityCollection = new EntityCollection();
        knownEntities = new HashMap<String, IndexerEntityData>();
    }

    private IndexerEntityData getEntityData(String egoEntityUriString, IndexerParameters indexParameters) throws URISyntaxException {
        URI egoEntityUri = new URI(egoEntityUriString);
        return getEntityData(egoEntityUri, indexParameters);
    }

    private IndexerEntityData getEntityData(URI egoEntityUri, IndexerParameters indexParameters) {
        IndexerEntityData entityData = knownEntities.get(egoEntityUri.toASCIIString());
        if (entityData != null) {
            return entityData;
        } else {
            entityData = new IndexerEntityData(null); // todo: while this could pass the identifier it is unlikely that this class will use them as it relies on the url instead
            knownEntities.put(egoEntityUri.toASCIIString(), entityData);
            try {
                Document linksDom = new CmdiComponentBuilder().getDocument(egoEntityUri);
                NodeList relationLinkNodeList = org.apache.xpath.XPathAPI.selectNodeList(linksDom, indexParameters.linkPath);
                for (int nodeCounter = 0; nodeCounter < relationLinkNodeList.getLength(); nodeCounter++) {
                    Node relationLinkNode = relationLinkNodeList.item(nodeCounter);
                    if (relationLinkNode != null) {
                        // resolve the alter URL against its ego URI
                        URI alterUri = egoEntityUri.resolve(relationLinkNode.getTextContent());
                        entityData.addRelation(alterUri.toASCIIString());
                        // get any requested link data
                        for (ParameterElement relevantDataPath : indexParameters.relevantLinkData.getValues()) {
                            for (Node linkDataNode = relationLinkNode.getParentNode().getFirstChild(); linkDataNode != null; linkDataNode = linkDataNode.getNextSibling()) {
                                if (relevantDataPath.getXpathString().equals(linkDataNode.getNodeName())) {
                                    entityData.addRelationData(alterUri.toASCIIString(), relevantDataPath.getXpathString(), linkDataNode.getTextContent());
                                }
                            }
                        }
                    }
                }
                // get any requested entity data
                // todo: this has been removed due to changes in the data storage and would have to be replaced if this class comes back into use
//                for (String[] relevantDataPath : indexParameters.getRelevantEntityData()) {
//                    NodeList relevantaDataNodeList = org.apache.xpath.XPathAPI.selectNodeList(linksDom, relevantDataPath[0]);
//                    for (int dataCounter = 0; dataCounter < relevantaDataNodeList.getLength(); dataCounter++) {
//                        Node dataNode = relevantaDataNodeList.item(dataCounter);
//                        if (dataNode != null) {
//                            entityData.addEntityData(relevantDataPath[0], dataNode.getTextContent());
//                        }
//                    }
//                }
            } catch (TransformerException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            } catch (ParserConfigurationException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            } catch (DOMException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            } catch (IOException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            } catch (SAXException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
            }
            return entityData;
        }
    }

    public void printKnownEntities() {
        for (String currentEgo : knownEntities.keySet()) {
            System.out.println("currentEgo: " + currentEgo);
            IndexerEntityData currentEntityData = knownEntities.get(currentEgo);
            for (String[] currentRecord : currentEntityData.getEntityFields()) {
                System.out.println("-> entityField: " + currentRecord[0] + " : " + currentRecord[1]);
            }
            for (String currentLink : currentEntityData.getRelationPaths()) {
                System.out.println("--> currentLink: " + currentLink);
                for (String[] currentRecord : currentEntityData.getRelationData(currentLink)) {
                    System.out.println("---> linkField: " + currentRecord[0] + " : " + currentRecord[1]);
                }
            }
        }
    }

    public void loadAllEntities(IndexerParameters indexParameters) {
        String[] treeNodesArray = LinorgSessionStorage.getSingleInstance().loadStringArray("KinGraphTree");
        if (treeNodesArray != null) {
            for (String currentNodeString : treeNodesArray) {
                try {
                    URI egoEntityUri = new URI(currentNodeString);
                    getEntityData(egoEntityUri, indexParameters);
                } catch (URISyntaxException exception) {
                    GuiHelper.linorgBugCatcher.logError(exception);
                }
            }
        }
    }

    public void setKinTypeStringTerm(String symbolString, String fieldPath, String fieldValue) {
        // todo: set the terms that combine to form the kin type strings
        // eg: setKinTypeStringTerm("M", "Kinnate.Gedcom.Entity.SEX", "F");
    }

    private EntityData getGraphDataNode(boolean isEgo, URI entityUri, IndexerParameters indexParameters) {
        IndexerEntityData entityData = getEntityData(entityUri, indexParameters);
        ArrayList<String> labelTextList = new ArrayList<String>();
        for (ParameterElement currentLabelField : indexParameters.labelFields.getValues()) {
            String labelTextTemp = entityData.getEntityField(currentLabelField.getXpathString());
            if (labelTextTemp != null) {
                labelTextList.add(labelTextTemp);
            }
        }
        for (ParameterElement currentSymbolField : indexParameters.symbolFieldsFields.getValues()) {
            String linkSymbolString = entityData.getEntityField(currentSymbolField.getXpathString());
            if (linkSymbolString != null) {
                return new EntityData(entityData.getUniqueIdentifier(), entityUri.toASCIIString(), null, currentSymbolField.getSelectedValue(), labelTextList.toArray(new String[]{}), isEgo);
            }
        }
        return new EntityData(entityData.getUniqueIdentifier(), entityUri.toASCIIString(), null, EntityData.SymbolType.none, labelTextList.toArray(new String[]{}), isEgo);
    }

    private void setRelationData(EntityData egoNode, EntityData alterNode, IndexerEntityData egoData, String alterPath, IndexerParameters indexParameters) {
        DataTypes.RelationType egoType = null;
        DataTypes.RelationType alterType = null;
        String[][] alterRelationFields = egoData.getRelationData(alterPath);
        if (alterRelationFields != null) {
            for (ParameterElement ancestorField : indexParameters.ancestorFields.getValues()) {
                for (String[] egoRelationField : alterRelationFields) {
                    if (ancestorField.getXpathString().equals(egoRelationField)) {
                        egoType = DataTypes.RelationType.ancestor;
                        alterType = DataTypes.RelationType.descendant;
                    }
                }
            }
            for (ParameterElement ancestorField : indexParameters.decendantFields.getValues()) {
                for (String[] egoRelationField : alterRelationFields) {
                    if (ancestorField.getXpathString().equals(egoRelationField[1])) {
{'fraction_non_alphanumeric': 0.08955638221087113, 'fraction_numerical': 0.003109211037699184, 'mean_word_length': 2.495439549776829, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3137692', 'n_tokens_mistral': 4647, 'n_tokens_neox': 4456, 'n_words': 923}