<commit_before>package nl.mpi.arbil.clarin;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.UUID;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.TransformerFactoryConfigurationError;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import nl.mpi.arbil.GuiHelper;
import nl.mpi.arbil.LinorgSessionStorage;
import nl.mpi.arbil.LinorgWindowManager;
import nl.mpi.arbil.data.ImdiTreeObject;
import org.apache.xmlbeans.SchemaProperty;
import org.apache.xmlbeans.SchemaType;
import org.apache.xmlbeans.SchemaTypeSystem;
import org.apache.xmlbeans.XmlBeans;
import org.apache.xmlbeans.XmlException;
import org.apache.xmlbeans.XmlObject;
import org.apache.xmlbeans.XmlOptions;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

/**
 * Document   : CmdiComponentBuilder
 * Created on : Mar 18, 2010, 1:40:35 PM
 * @author <EMAIL>
 */
public class CmdiComponentBuilder {

    private Document getDocument(File inputFile) throws ParserConfigurationException, SAXException, IOException {
        DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
        documentBuilderFactory.setValidating(false);
        documentBuilderFactory.setNamespaceAware(true);
        DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
        Document document;
        if (inputFile == null) {
            document = documentBuilder.newDocument();
        } else {
            document = documentBuilder.parse(inputFile);
        }
        return document;
    }

//    private Document createDocument(File inputFile) throws ParserConfigurationException, SAXException, IOException {
//        DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
//        documentBuilderFactory.setValidating(false);
//        DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
//        Document document = documentBuilder.newDocument();
//        return document;
//    }
    private void savePrettyFormatting(Document document, File outputFile) {
        try {
            // set up input and output
            DOMSource dOMSource = new DOMSource(document);
            FileOutputStream fileOutputStream = new FileOutputStream(outputFile);
            StreamResult xmlOutput = new StreamResult(fileOutputStream);
            // configure transformer
            Transformer transformer = TransformerFactory.newInstance().newTransformer();
            //transformer.setOutputProperty(OutputKeys.DOCTYPE_SYSTEM, "testing.dtd");
            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
            transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
            transformer.transform(dOMSource, xmlOutput);
            xmlOutput.getOutputStream().close();
            //System.out.println(xmlOutput.getWriter().toString());
        } catch (IllegalArgumentException illegalArgumentException) {
            GuiHelper.linorgBugCatcher.logError(illegalArgumentException);
        } catch (TransformerException transformerException) {
            GuiHelper.linorgBugCatcher.logError(transformerException);
        } catch (TransformerFactoryConfigurationError transformerFactoryConfigurationError) {
            System.out.println(transformerFactoryConfigurationError.getMessage());
        } catch (FileNotFoundException notFoundException) {
            GuiHelper.linorgBugCatcher.logError(notFoundException);
        } catch (IOException iOException) {
            GuiHelper.linorgBugCatcher.logError(iOException);
        }
    }

    public URI insertResourceProxy(ImdiTreeObject targetNode, ImdiTreeObject resourceNode) {
//    <.CMD.Resources.ResourceProxyList.ResourceProxy>
//        <ResourceProxyList>
//            <ResourceProxy id="a_text">
//                <ResourceType>Resource</ResourceType>
//                <ResourceRef>bla.txt</ResourceRef>
//            </ResourceProxy>
        String targetXmlPath = targetNode.getURI().getFragment();
        System.out.println("insertResourceProxy: " + targetXmlPath);
        File cmdiNodeFile = targetNode.getFile();
        String nodeFragment = "";

        // geerate a uuid for new resource
        String resourceProxyId = UUID.randomUUID().toString();
        try {
            // load the schema
            SchemaType schemaType = getFirstSchemaType(targetNode.getNodeTemplate().templateFile);
            // load the dom
            Document targetDocument = getDocument(cmdiNodeFile);
            // insert the new section
            try {
                try {
//                    if (targetXmlPath == null) {
//                        targetXmlPath = ".CMD.Components";
//                    }
                    Node documentNode = selectSingleNode(targetDocument, targetXmlPath);
                    documentNode.getAttributes().getNamedItem("ref").setNodeValue(resourceProxyId);
                } catch (TransformerException exception) {
                    GuiHelper.linorgBugCatcher.logError(exception);
                    return null;
                }
//                printoutDocument(targetDocument);
                Node addedResourceNode = insertSectionToXpath(targetDocument, targetDocument.getFirstChild(), schemaType, ".CMD.Resources.ResourceProxyList", ".CMD.Resources.ResourceProxyList.ResourceProxy");
                addedResourceNode.getAttributes().getNamedItem("id").setNodeValue(resourceProxyId);
                NodeList childList = addedResourceNode.getChildNodes();
                // this depends on the node order in the schema, which is a fallible way to do this, there should be some sort of check that we really have the correct nodes here
                Node resourceTypeNode = childList.item(1);
                Node resourceRefNode = childList.item(2);
                System.out.println("addedResourceNode: " + addedResourceNode.getLocalName());
                System.out.println("resourceTypeNode: " + resourceTypeNode.getLocalName());
                System.out.println("resourceRefNode: " + resourceRefNode.getLocalName());
                resourceTypeNode.setNodeValue(resourceNode.mpiMimeType);
                resourceRefNode.setNodeValue(resourceNode.getUrlString());
            } catch (Exception exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
                return null;
            }
            // bump the history
            targetNode.bumpHistory();
            // save the dom
            savePrettyFormatting(targetDocument, cmdiNodeFile); // note that we want to make sure that this gets saved even without changes because we have bumped the history ant there will be no file otherwise
        } catch (IOException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        } catch (ParserConfigurationException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        } catch (SAXException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        }
        return targetNode.getURI();
    }

    public URI insertChildComponent(ImdiTreeObject targetNode, String targetXmlPath, String cmdiComponentId) {
        System.out.println("insertChildComponent: " + cmdiComponentId);
        System.out.println("targetXmlPath: " + targetXmlPath);
        if (targetXmlPath == null) {
            targetXmlPath = cmdiComponentId.replaceAll("\\.[^.]+$", "");
        } else if (targetXmlPath.length() == cmdiComponentId.length()) {
            targetXmlPath = targetXmlPath.replaceAll("\\.[^.]+$", "");
        }
        System.out.println("trimmed targetXmlPath: " + targetXmlPath);
        //String targetXpath = targetNode.getURI().getFragment();
        //System.out.println("targetXpath: " + targetXpath);
        File cmdiNodeFile = targetNode.getFile();
        String nodeFragment = "";
        try {
            // load the schema
            SchemaType schemaType = getFirstSchemaType(targetNode.getNodeTemplate().templateFile);
            // load the dom
            Document targetDocument = getDocument(cmdiNodeFile);
            // insert the new section
            try {
//                printoutDocument(targetDocument);
                Node AddedNode = insertSectionToXpath(targetDocument, targetDocument.getFirstChild(), schemaType, targetXmlPath, cmdiComponentId);
                nodeFragment = convertNodeToNodePath(targetDocument, AddedNode);
            } catch (Exception exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
                return null;
            }
            // bump the history
            targetNode.bumpHistory();
            // save the dom
            savePrettyFormatting(targetDocument, cmdiNodeFile); // note that we want to make sure that this gets saved even without changes because we have bumped the history ant there will be no file otherwise
        } catch (IOException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        } catch (ParserConfigurationException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        } catch (SAXException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        }
//       diff_match_patch diffTool= new diff_match_patch();
//       diffTool.diff_main(targetXpath, targetXpath);
        try {
            System.out.println("nodeFragment: " + nodeFragment);
            // return the child node url and path in the xml
            // first strip off any fragment then add the full node fragment
            return new URI(targetNode.getURI().toString().split("#")[0] + "#" + nodeFragment);
        } catch (URISyntaxException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        }
    }

    private Node selectSingleNode(Document targetDocument, String targetXpath) throws TransformerException {
        // convert the syntax inherited from the imdi api into xpath
        String tempXpath = targetXpath.replaceAll("\\.", "/:");
        tempXpath = tempXpath.replaceAll("\\(", "[");
        tempXpath = tempXpath.replaceAll("\\)", "]");
//            tempXpath = "/CMD/Components/Session/MDGroup/Actors";
        System.out.println("tempXpath: " + tempXpath);
        // find the target node of the xml
        return org.apache.xpath.XPathAPI.selectSingleNode(targetDocument, tempXpath);
    }

    private Node insertSectionToXpath(Document targetDocument, Node documentNode, SchemaType schemaType, String targetXpath, String xsdPath) throws Exception {
        System.out.println("insertSectionToXpath");
        System.out.println("xsdPath: " + xsdPath);
        System.out.println("targetXpath: " + targetXpath);
        SchemaProperty foundProperty = null;
        String strippedXpath = null;
        if (targetXpath == null) {
            documentNode = documentNode.getParentNode();
        } else {
            try {
                documentNode = selectSingleNode(targetDocument, targetXpath);
            } catch (TransformerException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
                return null;
            }
            strippedXpath = targetXpath.replaceAll("\\(\\d+\\)", "");
        }
        // at this point we have the xml node that the user acted on but next must get any additional nodes with the next section        
        System.out.println("strippedXpath: " + strippedXpath);
        for (String currentPathComponent : xsdPath.split("\\.")) {
            if (currentPathComponent.length() > 0) {
                System.out.println("currentPathComponent: " + currentPathComponent);
                //System.out.println("documentNode: " + documentNode.getChildNodes());
                // get the starting point in the schema
                foundProperty = null;
                for (SchemaProperty schemaProperty : schemaType.getProperties()) {
                    String currentName = schemaProperty.getName().getLocalPart();
                    if (currentPathComponent.equals(currentName)) {
                        foundProperty = schemaProperty;
                        break;
                    }
                }
                if (foundProperty == null) {
                    throw new Exception("failed to find the path in the schema: " + currentPathComponent);
                } else {
                    schemaType = foundProperty.getType();
                    System.out.println("foundProperty: " + foundProperty.getName().getLocalPart());
                }
                // get the starting node in the xml document
                // TODO: this will not navigate to the nth child node when the xpath is provided
                //if (foundNode != null) {
                //    // keep the last node found in the chain

//                if (strippedXpath != null && strippedXpath.startsWith("." + currentPathComponent)) {
//                    strippedXpath = strippedXpath.substring(currentPathComponent.length() + 1);
//                    System.out.println("strippedXpath: " + strippedXpath);
//                } else {
//                    Node childNode = documentNode.getFirstChild();
//                    while (childNode != null) {
//                        System.out.println("childNode: " + childNode.getNodeName());
//                        if (currentPathComponent.equals(childNode.getNodeName())) {
//                            System.out.println("found existing: " + currentPathComponent);
//                            documentNode = childNode;
//                            break;
//                        } else {
//                            childNode = childNode.getNextSibling();
//                        }
//                    }
//                    if (documentNode != childNode) {
//                        System.out.println("Adding destination node: " + currentPathComponent);
//                        System.out.println("Into: " + documentNode.getNodeName());
//                        documentNode = (Node) appendNode(targetDocument, null, documentNode, foundProperty);
////                        throw new Exception("failed to find the node path in the document: " + currentPathComponent);
//                    }
//                }
            }
        }
//        System.out.println("Adding marker node");
//        Element currentElement = targetDocument.createElement("MarkerNode");
//        currentElement.setTextContent(xsdPath);
//        documentNode.appendChild(currentElement);
        System.out.println("Adding destination sub nodes node to: " + documentNode.getLocalName());
        constructXml(foundProperty.getType().getOuterType(), xsdPath, targetDocument, null, documentNode, true);
        return documentNode;
    }

    private String convertNodeToNodePath(Document targetDocument, Node documentNode) {
        System.out.println("Calculating the added fragment");
        String nodeFragment = documentNode.getNodeName();
        Node parentNode = documentNode.getParentNode();
        while (parentNode != null) {
            // TODO: handle sibbling node counts
            // count siblings to get the correct child index for the fragment
            System.out.println("nodeFragment: " + nodeFragment);
            nodeFragment = parentNode.getNodeName() + "." + nodeFragment;
            if (parentNode.isSameNode(targetDocument.getDocumentElement())) {
                break;
            }
            parentNode = parentNode.getParentNode();
        }
        nodeFragment = "." + nodeFragment;
        System.out.println("nodeFragment: " + nodeFragment);
        return nodeFragment;
        //return childStartElement.
    }

    public URI createComponentFile(URI cmdiNodeFile, URI xsdFile) {
        System.out.println("createComponentFile: " + cmdiNodeFile + " : " + xsdFile);
        try {
            Document workingDocument = getDocument(null);
            readSchema(workingDocument, xsdFile);
            savePrettyFormatting(workingDocument, new File(cmdiNodeFile));
        } catch (IOException e) {
            GuiHelper.linorgBugCatcher.logError(e);
        } catch (ParserConfigurationException e) {
            GuiHelper.linorgBugCatcher.logError(e);
        } catch (SAXException e) {
            GuiHelper.linorgBugCatcher.logError(e);
        }
        return cmdiNodeFile;
    }

    private SchemaType getFirstSchemaType(File schemaFile) {
        try {
            InputStream inputStream = new FileInputStream(schemaFile);
            //Since we're dealing with xml schema files here the character encoding is assumed to be UTF-8
            XmlOptions options = new XmlOptions();
            options.setCharacterEncoding("UTF-8");
            SchemaTypeSystem sts = XmlBeans.compileXsd(new XmlObject[]{XmlObject.Factory.parse(inputStream, options)}, XmlBeans.getBuiltinTypeSystem(), null);
            // there can only be a single root node so we just get the first one, note that the IMDI schema specifies two (METATRANSCRIPT and VocabularyDef)
            return sts.documentTypes()[0];
        } catch (IOException e) {
            GuiHelper.linorgBugCatcher.logError(e);
        } catch (XmlException e) {
            // TODO: this is not really a good place to message this so modify to throw
            LinorgWindowManager.getSingleInstance().addMessageDialogToQueue("Could not read the XML Schema", "Error inserting node");
            GuiHelper.linorgBugCatcher.logError(e);
        }
        return null;
    }

    private void readSchema(Document workingDocument, URI xsdFile) {
        File schemaFile = LinorgSessionStorage.getSingleInstance().updateCache(xsdFile.toString(), 5);
        SchemaType schemaType = getFirstSchemaType(schemaFile);
        constructXml(schemaType, "documentTypes", workingDocument, xsdFile.toString(), null, false);
    }

    private Element appendNode(Document workingDocument, String nameSpaceUri, Node parentElement, SchemaProperty schemaProperty) {
        Element currentElement = workingDocument.createElementNS("http://www.clarin.eu/cmd", schemaProperty.getName().getLocalPart());
        SchemaType currentSchemaType = schemaProperty.getType();
        for (SchemaProperty attributesProperty : currentSchemaType.getAttributeProperties()) {
            currentElement.setAttribute(attributesProperty.getName().getLocalPart(), attributesProperty.getDefaultText());
        }
        if (parentElement == null) {
            // this is probably not the way to set these, however this will do for now (many other methods have been tested and all failed to function correctly)
            currentElement.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
            currentElement.setAttribute("xsi:schemaLocation", "http://www.clarin.eu/cmd " + nameSpaceUri);
            workingDocument.appendChild(currentElement);
        } else {
            parentElement.appendChild(currentElement);
        }
        //currentElement.setTextContent(schemaProperty.getMinOccurs() + ":" + schemaProperty.getMinOccurs());
        return currentElement;
    }

    private void constructXml(SchemaType schemaType, String pathString, Document workingDocument, String nameSpaceUri, Node parentElement, boolean forceFirstNode) {
        //System.out.println("printSchemaType " + schemaType.toString());
//        for (SchemaType schemaSubType : schemaType.getAnonymousTypes()) {
//            System.out.println("getAnonymousTypes:");
//            constructXml(schemaSubType, pathString + ".*getAnonymousTypes*", workingDocument, parentElement);
//        }
//        for (SchemaType schemaSubType : schemaType.getUnionConstituentTypes()) {
//            System.out.println("getUnionConstituentTypes:");
//            printSchemaType(schemaSubType);
//        }
//        for (SchemaType schemaSubType : schemaType.getUnionMemberTypes()) {
//            System.out.println("getUnionMemberTypes:");
//            constructXml(schemaSubType, pathString + ".*getUnionMemberTypes*", workingDocument, parentElement);
//        }
//        for (SchemaType schemaSubType : schemaType.getUnionSubTypes()) {
//            System.out.println("getUnionSubTypes:");
//            printSchemaType(schemaSubType);
//        }
        //SchemaType childType =schemaType.

        /////////////////////

        for (SchemaProperty schemaProperty : schemaType.getElementProperties()) {
            //for (int childCounter = 0; childCounter < schemaProperty.getMinOccurs().intValue(); childCounter++) {
            // if the searched element is a child node of the given node return
            // its SchemaType
            //if (properties[i].getName().toString().equals(element)) {
            String currentPathString = pathString + "." + schemaProperty.getName().getLocalPart();
            System.out.println("Found Element: " + currentPathString);
            SchemaType currentSchemaType = schemaProperty.getType();
            // if the searched element was not a child of the given Node
            // then again for each of these child nodes search recursively in
            // their child nodes, in the case they are a complex type, because
            // only complex types have child nodes
            //currentSchemaType.getAttributeProperties();
            if (forceFirstNode || schemaProperty.getMinOccurs() != BigInteger.ZERO) {
                forceFirstNode = false;
                Element currentElement = appendNode(workingDocument, nameSpaceUri, parentElement, schemaProperty);
                //     if ((schemaProperty.getType() != null) && (!(currentSchemaType.isSimpleType()))) {
                constructXml(currentSchemaType, currentPathString, workingDocument, nameSpaceUri, currentElement, false);
                //     }
            }
            //}
        }
    }

    private void printoutDocument(Document workingDocument) {
        try {
            TransformerFactory tranFactory = TransformerFactory.newInstance();
            Transformer transformer = tranFactory.newTransformer();
            transformer.setOutputProperty(OutputKeys.INDENT, "yes");<commit_msg>Updated the application title text in the windows installer. Modified the schema validation method (Check Format) to first use the name space url defined in the xml and then fall back to the imdi schema and then the schema provided in the jar file. Added clarin XSD reader. Loaded Clarin templates from the schema location of cmdi files. Addded component insertion into clarin files (via add menu). Added auto recovery from history when a file is missing. Updated the add menu contents to handle clarin sub nodes. Corrected nodes left after a delete from the tree. Updated the sub child node detection when reading in the metadata xml to handle cmdi templates with full paths. Added clarin link reader. Inserted Clarin linked nodes into the Arbil tree. Stored all clarin profile schemas on disk when they are first seen. Added Handle resolution and general URL resolution in Clarin links. Added test fields to the wizard html. Added CMDI function to tell when a resource can be added or not to a specific node. Set the profile name in clarin templates. Added test for drag and drop onto clarin nodes. Added resource drop action to add resource files to CMDI nodes. Set the drag cursor to indicate allow drops to CMDI nodes based on the values in the schema. Made more robust against broken cmdi resource links. Added resource proxy nodes for CMDI files when adding resources.<commit_after>package nl.mpi.arbil.clarin;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.UUID;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.TransformerFactoryConfigurationError;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import nl.mpi.arbil.GuiHelper;
import nl.mpi.arbil.LinorgSessionStorage;
import nl.mpi.arbil.LinorgWindowManager;
import nl.mpi.arbil.data.ImdiTreeObject;
import org.apache.xmlbeans.SchemaProperty;
import org.apache.xmlbeans.SchemaType;
import org.apache.xmlbeans.SchemaTypeSystem;
import org.apache.xmlbeans.XmlBeans;
import org.apache.xmlbeans.XmlException;
import org.apache.xmlbeans.XmlObject;
import org.apache.xmlbeans.XmlOptions;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.xml.sax.SAXException;

/**
 * Document   : CmdiComponentBuilder
 * Created on : Mar 18, 2010, 1:40:35 PM
 * @author <EMAIL>
 */
public class CmdiComponentBuilder {

    private Document getDocument(File inputFile) throws ParserConfigurationException, SAXException, IOException {
        DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
        documentBuilderFactory.setValidating(false);
        documentBuilderFactory.setNamespaceAware(true);
        DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
        Document document;
        if (inputFile == null) {
            document = documentBuilder.newDocument();
        } else {
            document = documentBuilder.parse(inputFile);
        }
        return document;
    }

//    private Document createDocument(File inputFile) throws ParserConfigurationException, SAXException, IOException {
//        DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
//        documentBuilderFactory.setValidating(false);
//        DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
//        Document document = documentBuilder.newDocument();
//        return document;
//    }
    private void savePrettyFormatting(Document document, File outputFile) {
        try {
            // set up input and output
            DOMSource dOMSource = new DOMSource(document);
            FileOutputStream fileOutputStream = new FileOutputStream(outputFile);
            StreamResult xmlOutput = new StreamResult(fileOutputStream);
            // configure transformer
            Transformer transformer = TransformerFactory.newInstance().newTransformer();
            //transformer.setOutputProperty(OutputKeys.DOCTYPE_SYSTEM, "testing.dtd");
            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
            transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
            transformer.transform(dOMSource, xmlOutput);
            xmlOutput.getOutputStream().close();
            //System.out.println(xmlOutput.getWriter().toString());
        } catch (IllegalArgumentException illegalArgumentException) {
            GuiHelper.linorgBugCatcher.logError(illegalArgumentException);
        } catch (TransformerException transformerException) {
            GuiHelper.linorgBugCatcher.logError(transformerException);
        } catch (TransformerFactoryConfigurationError transformerFactoryConfigurationError) {
            System.out.println(transformerFactoryConfigurationError.getMessage());
        } catch (FileNotFoundException notFoundException) {
            GuiHelper.linorgBugCatcher.logError(notFoundException);
        } catch (IOException iOException) {
            GuiHelper.linorgBugCatcher.logError(iOException);
        }
    }

    public URI insertResourceProxy(ImdiTreeObject targetNode, ImdiTreeObject resourceNode) {
//    <.CMD.Resources.ResourceProxyList.ResourceProxy>
//        <ResourceProxyList>
//            <ResourceProxy id="a_text">
//                <ResourceType>Resource</ResourceType>
//                <ResourceRef>bla.txt</ResourceRef>
//            </ResourceProxy>
        String targetXmlPath = targetNode.getURI().getFragment();
        System.out.println("insertResourceProxy: " + targetXmlPath);
        File cmdiNodeFile = targetNode.getFile();
        String nodeFragment = "";

        // geerate a uuid for new resource
        String resourceProxyId = UUID.randomUUID().toString();
        try {
            // load the schema
            SchemaType schemaType = getFirstSchemaType(targetNode.getNodeTemplate().templateFile);
            // load the dom
            Document targetDocument = getDocument(cmdiNodeFile);
            // insert the new section
            try {
                try {
//                    if (targetXmlPath == null) {
//                        targetXmlPath = ".CMD.Components";
//                    }
                    Node documentNode = selectSingleNode(targetDocument, targetXmlPath);
                    documentNode.getAttributes().getNamedItem("ref").setNodeValue(resourceProxyId);
                } catch (TransformerException exception) {
                    GuiHelper.linorgBugCatcher.logError(exception);
                    return null;
                }
//                printoutDocument(targetDocument);
                Node addedResourceNode = insertSectionToXpath(targetDocument, targetDocument.getFirstChild(), schemaType, ".CMD.Resources.ResourceProxyList", ".CMD.Resources.ResourceProxyList.ResourceProxy");
                addedResourceNode.getAttributes().getNamedItem("id").setNodeValue(resourceProxyId);
                for (Node childNode = addedResourceNode.getFirstChild(); childNode != null; childNode = childNode.getNextSibling()) {
                    String localName = childNode.getLocalName();
                    if ("ResourceType".equals(localName)) {
                        childNode.setTextContent(resourceNode.mpiMimeType);
                    }
                    if ("ResourceRef".equals(localName)) {
                        childNode.setTextContent(resourceNode.getUrlString());
                    }
                }
            } catch (Exception exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
                return null;
            }
            // bump the history
            targetNode.bumpHistory();
            // save the dom
            savePrettyFormatting(targetDocument, cmdiNodeFile); // note that we want to make sure that this gets saved even without changes because we have bumped the history ant there will be no file otherwise
        } catch (IOException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        } catch (ParserConfigurationException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        } catch (SAXException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        }
        return targetNode.getURI();
    }

    public URI insertChildComponent(ImdiTreeObject targetNode, String targetXmlPath, String cmdiComponentId) {
        System.out.println("insertChildComponent: " + cmdiComponentId);
        System.out.println("targetXmlPath: " + targetXmlPath);
        if (targetXmlPath == null) {
            targetXmlPath = cmdiComponentId.replaceAll("\\.[^.]+$", "");
        } else if (targetXmlPath.length() == cmdiComponentId.length()) {
            targetXmlPath = targetXmlPath.replaceAll("\\.[^.]+$", "");
        }
        System.out.println("trimmed targetXmlPath: " + targetXmlPath);
        //String targetXpath = targetNode.getURI().getFragment();
        //System.out.println("targetXpath: " + targetXpath);
        File cmdiNodeFile = targetNode.getFile();
        String nodeFragment = "";
        try {
            // load the schema
            SchemaType schemaType = getFirstSchemaType(targetNode.getNodeTemplate().templateFile);
            // load the dom
            Document targetDocument = getDocument(cmdiNodeFile);
            // insert the new section
            try {
//                printoutDocument(targetDocument);
                Node AddedNode = insertSectionToXpath(targetDocument, targetDocument.getFirstChild(), schemaType, targetXmlPath, cmdiComponentId);
                nodeFragment = convertNodeToNodePath(targetDocument, AddedNode);
            } catch (Exception exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
                return null;
            }
            // bump the history
            targetNode.bumpHistory();
            // save the dom
            savePrettyFormatting(targetDocument, cmdiNodeFile); // note that we want to make sure that this gets saved even without changes because we have bumped the history ant there will be no file otherwise
        } catch (IOException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        } catch (ParserConfigurationException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        } catch (SAXException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        }
//       diff_match_patch diffTool= new diff_match_patch();
//       diffTool.diff_main(targetXpath, targetXpath);
        try {
            System.out.println("nodeFragment: " + nodeFragment);
            // return the child node url and path in the xml
            // first strip off any fragment then add the full node fragment
            return new URI(targetNode.getURI().toString().split("#")[0] + "#" + nodeFragment);
        } catch (URISyntaxException exception) {
            GuiHelper.linorgBugCatcher.logError(exception);
            return null;
        }
    }

    private Node selectSingleNode(Document targetDocument, String targetXpath) throws TransformerException {
        // convert the syntax inherited from the imdi api into xpath
        String tempXpath = targetXpath.replaceAll("\\.", "/:");
        tempXpath = tempXpath.replaceAll("\\(", "[");
        tempXpath = tempXpath.replaceAll("\\)", "]");
//            tempXpath = "/CMD/Components/Session/MDGroup/Actors";
        System.out.println("tempXpath: " + tempXpath);
        // find the target node of the xml
        return org.apache.xpath.XPathAPI.selectSingleNode(targetDocument, tempXpath);
    }

    private Node insertSectionToXpath(Document targetDocument, Node documentNode, SchemaType schemaType, String targetXpath, String xsdPath) throws Exception {
        System.out.println("insertSectionToXpath");
        System.out.println("xsdPath: " + xsdPath);
        System.out.println("targetXpath: " + targetXpath);
        SchemaProperty foundProperty = null;
        String strippedXpath = null;
        if (targetXpath == null) {
            documentNode = documentNode.getParentNode();
        } else {
            try {
                documentNode = selectSingleNode(targetDocument, targetXpath);
            } catch (TransformerException exception) {
                GuiHelper.linorgBugCatcher.logError(exception);
                return null;
            }
            strippedXpath = targetXpath.replaceAll("\\(\\d+\\)", "");
        }
        // at this point we have the xml node that the user acted on but next must get any additional nodes with the next section        
        System.out.println("strippedXpath: " + strippedXpath);
        for (String currentPathComponent : xsdPath.split("\\.")) {
            if (currentPathComponent.length() > 0) {
                System.out.println("currentPathComponent: " + currentPathComponent);
                //System.out.println("documentNode: " + documentNode.getChildNodes());
                // get the starting point in the schema
                foundProperty = null;
                for (SchemaProperty schemaProperty : schemaType.getProperties()) {
                    String currentName = schemaProperty.getName().getLocalPart();
                    if (currentPathComponent.equals(currentName)) {
                        foundProperty = schemaProperty;
                        break;
                    }
                }
                if (foundProperty == null) {
                    throw new Exception("failed to find the path in the schema: " + currentPathComponent);
                } else {
                    schemaType = foundProperty.getType();
                    System.out.println("foundProperty: " + foundProperty.getName().getLocalPart());
                }
                // get the starting node in the xml document
                // TODO: this will not navigate to the nth child node when the xpath is provided
                //if (foundNode != null) {
                //    // keep the last node found in the chain

//                if (strippedXpath != null && strippedXpath.startsWith("." + currentPathComponent)) {
//                    strippedXpath = strippedXpath.substring(currentPathComponent.length() + 1);
//                    System.out.println("strippedXpath: " + strippedXpath);
//                } else {
//                    Node childNode = documentNode.getFirstChild();
//                    while (childNode != null) {
//                        System.out.println("childNode: " + childNode.getNodeName());
//                        if (currentPathComponent.equals(childNode.getNodeName())) {
//                            System.out.println("found existing: " + currentPathComponent);
//                            documentNode = childNode;
//                            break;
//                        } else {
//                            childNode = childNode.getNextSibling();
//                        }
//                    }
//                    if (documentNode != childNode) {
//                        System.out.println("Adding destination node: " + currentPathComponent);
//                        System.out.println("Into: " + documentNode.getNodeName());
//                        documentNode = (Node) appendNode(targetDocument, null, documentNode, foundProperty);
////                        throw new Exception("failed to find the node path in the document: " + currentPathComponent);
//                    }
//                }
            }
        }
//        System.out.println("Adding marker node");
//        Element currentElement = targetDocument.createElement("MarkerNode");
//        currentElement.setTextContent(xsdPath);
//        documentNode.appendChild(currentElement);
        System.out.println("Adding destination sub nodes node to: " + documentNode.getLocalName());
        return constructXml(foundProperty.getType().getOuterType(), xsdPath, targetDocument, null, documentNode, true);
    }

    private String convertNodeToNodePath(Document targetDocument, Node documentNode) {
        System.out.println("Calculating the added fragment");
        String nodeFragment = documentNode.getNodeName();
        Node parentNode = documentNode.getParentNode();
        while (parentNode != null) {
            // TODO: handle sibbling node counts
            // count siblings to get the correct child index for the fragment
            System.out.println("nodeFragment: " + nodeFragment);
            nodeFragment = parentNode.getNodeName() + "." + nodeFragment;
            if (parentNode.isSameNode(targetDocument.getDocumentElement())) {
                break;
            }
            parentNode = parentNode.getParentNode();
        }
        nodeFragment = "." + nodeFragment;
        System.out.println("nodeFragment: " + nodeFragment);
        return nodeFragment;
        //return childStartElement.
    }

    public URI createComponentFile(URI cmdiNodeFile, URI xsdFile) {
        System.out.println("createComponentFile: " + cmdiNodeFile + " : " + xsdFile);
        try {
            Document workingDocument = getDocument(null);
            readSchema(workingDocument, xsdFile);
            savePrettyFormatting(workingDocument, new File(cmdiNodeFile));
        } catch (IOException e) {
            GuiHelper.linorgBugCatcher.logError(e);
        } catch (ParserConfigurationException e) {
            GuiHelper.linorgBugCatcher.logError(e);
        } catch (SAXException e) {
            GuiHelper.linorgBugCatcher.logError(e);
        }
        return cmdiNodeFile;
    }

    private SchemaType getFirstSchemaType(File schemaFile) {
        try {
            InputStream inputStream = new FileInputStream(schemaFile);
            //Since we're dealing with xml schema files here the character encoding is assumed to be UTF-8
            XmlOptions options = new XmlOptions();
            options.setCharacterEncoding("UTF-8");
            SchemaTypeSystem sts = XmlBeans.compileXsd(new XmlObject[]{XmlObject.Factory.parse(inputStream, options)}, XmlBeans.getBuiltinTypeSystem(), null);
            // there can only be a single root node so we just get the first one, note that the IMDI schema specifies two (METATRANSCRIPT and VocabularyDef)
            return sts.documentTypes()[0];
        } catch (IOException e) {
            GuiHelper.linorgBugCatcher.logError(e);
        } catch (XmlException e) {
            // TODO: this is not really a good place to message this so modify to throw
            LinorgWindowManager.getSingleInstance().addMessageDialogToQueue("Could not read the XML Schema", "Error inserting node");
            GuiHelper.linorgBugCatcher.logError(e);
        }
        return null;
    }

    private void readSchema(Document workingDocument, URI xsdFile) {
        File schemaFile = LinorgSessionStorage.getSingleInstance().updateCache(xsdFile.toString(), 5);
        SchemaType schemaType = getFirstSchemaType(schemaFile);
        constructXml(schemaType, "documentTypes", workingDocument, xsdFile.toString(), null, false);
    }

    private Element appendNode(Document workingDocument, String nameSpaceUri, Node parentElement, SchemaProperty schemaProperty) {
        Element currentElement = workingDocument.createElementNS("http://www.clarin.eu/cmd", schemaProperty.getName().getLocalPart());
        SchemaType currentSchemaType = schemaProperty.getType();
        for (SchemaProperty attributesProperty : currentSchemaType.getAttributeProperties()) {
            currentElement.setAttribute(attributesProperty.getName().getLocalPart(), attributesProperty.getDefaultText());
        }
        if (parentElement == null) {
            // this is probably not the way to set these, however this will do for now (many other methods have been tested and all failed to function correctly)
            currentElement.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
            currentElement.setAttribute("xsi:schemaLocation", "http://www.clarin.eu/cmd " + nameSpaceUri);
            workingDocument.appendChild(currentElement);
        } else {
            parentElement.appendChild(currentElement);
        }
        //currentElement.setTextContent(schemaProperty.getMinOccurs() + ":" + schemaProperty.getMinOccurs());
        return currentElement;
    }

    private Node constructXml(SchemaType schemaType, String pathString, Document workingDocument, String nameSpaceUri, Node parentElement, boolean forceFirstNode) {
        Node returnNode = null;
        //System.out.println("printSchemaType " + schemaType.toString());
//        for (SchemaType schemaSubType : schemaType.getAnonymousTypes()) {
//            System.out.println("getAnonymousTypes:");
//            constructXml(schemaSubType, pathString + ".*getAnonymousTypes*", workingDocument, parentElement);
//        }
//        for (SchemaType schemaSubType : schemaType.getUnionConstituentTypes()) {
//            System.out.println("getUnionConstituentTypes:");
//            printSchemaType(schemaSubType);
//        }
//        for (SchemaType schemaSubType : schemaType.getUnionMemberTypes()) {
//            System.out.println("getUnionMemberTypes:");
//            constructXml(schemaSubType, pathString + ".*getUnionMemberTypes*", workingDocument, parentElement);
//        }
//        for (SchemaType schemaSubType : schemaType.getUnionSubTypes()) {
//            System.out.println("getUnionSubTypes:");
//            printSchemaType(schemaSubType);
//        }
        //SchemaType childType =schemaType.

        /////////////////////

        for (SchemaProperty schemaProperty : schemaType.getElementProperties()) {
            //for (int childCounter = 0; childCounter < schemaProperty.getMinOccurs().intValue(); childCounter++) {
            // if the searched element is a child node of the given node return
            // its SchemaType
            //if (properties[i].getName().toString().equals(element)) {
            String currentPathString = pathString + "." + schemaProperty.getName().getLocalPart();
            System.out.println("Found Element: " + currentPathString);
            SchemaType currentSchemaType = schemaProperty.getType();
            // if the searched element was not a child of the given Node
            // then again for each of these child nodes search recursively in
            // their child nodes, in the case they are a complex type, because
            // only complex types have child nodes
            //currentSchemaType.getAttributeProperties();
            if (forceFirstNode || schemaProperty.getMinOccurs() != BigInteger.ZERO) {
                forceFirstNode = false;
                Element currentElement = appendNode(workingDocument, nameSpaceUri, parentElement, schemaProperty);
                returnNode = currentElement;
                //     if ((schemaProperty.getType() != null) && (!(currentSchemaType.isSimpleType()))) {
                constructXml(currentSchemaType, currentPathString, workingDocument, nameSpaceUri, currentElement, false);
                //     }
            }
            //}
        }
        return returnNode;
    }

    private void printoutDocument(Document workingDocument) {
        try {
            TransformerFactory tranFactory = TransformerFactory.newInstance();
            Transformer transformer = tranFactory.newTransformer();
            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
{'fraction_non_alphanumeric': 0.09596152216289516, 'fraction_numerical': 0.001250105941181456, 'mean_word_length': 2.489095882309455, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 64}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1079562', 'n_tokens_mistral': 11574, 'n_tokens_neox': 11171, 'n_words': 2720}