<commit_before>/*
 * eXist Open Source Native XML Database
 * Copyright (C) 2001-2017 The eXist Project
 * http://exist-db.org
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
package org.exist.xquery.functions.transform;

import com.evolvedbinary.j8fu.tuple.Tuple2;
import org.exist.EXistException;
import org.exist.collections.Collection;
import org.exist.collections.IndexInfo;
import org.exist.collections.triggers.TriggerException;
import org.exist.security.PermissionDeniedException;
import org.exist.storage.BrokerPool;
import org.exist.storage.DBBroker;
import org.exist.storage.lock.Lock;
import org.exist.storage.lock.ManagedCollectionLock;
import org.exist.storage.txn.Txn;
import org.exist.test.ExistEmbeddedServer;
import org.exist.util.LockException;
import org.exist.xmldb.XmldbURI;
import org.exist.xquery.XPathException;
import org.exist.xquery.XQuery;
import org.exist.xquery.value.Item;
import org.exist.xquery.value.Sequence;
import org.exist.xquery.value.Type;
import org.junit.*;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import org.xmlunit.builder.DiffBuilder;
import org.xmlunit.builder.Input;
import org.xmlunit.diff.Diff;

import javax.xml.transform.Source;
import java.io.IOException;
import java.util.Optional;

import static com.evolvedbinary.j8fu.tuple.Tuple.Tuple;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNotNull;

/**
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 */
public class TransformTest {

    @ClassRule
    public static ExistEmbeddedServer existEmbeddedServer = new ExistEmbeddedServer(true, true);

    private static final XmldbURI TEST_IDS_COLLECTION = XmldbURI.create("/db/transform-ids-test");

    private static final XmldbURI INPUT_LIST_XML_NAME = XmldbURI.create("inputListOps.xml");

    private static final String INPUT_XML =
            "<listOps>\n" +
            "    <ops id=\"IRCANTEC\"/>\n" +
            "    <ops id=\"CIBTP\"/>\n" +
            "    <ops id=\"AGIRC-ARRCO\"/>\n" +
            "    <ops id=\"CTIP-FFSA-FNMF\"/>\n" +
            "</listOps>";

    private static final XmldbURI DICTIONARY_XML_NAME = XmldbURI.create("listOpsErr.xml");

    private static final String DICTIONARY_XML =
            "<listOps>\n" +
            "    <ops id=\"IRCANTEC\" doEntiteAff=\"false\" doGenerateB20=\"true\"> </ops>\n" +
            "    <ops id=\"CIBTP\" doEntiteAff=\"true\" doGenerateB20=\"true\"/>\n" +
            "    <ops id=\"AGIRC-ARRCO\" doEntiteAff=\"true\" doGenerateB20=\"false\"> </ops>\n" +
            "    <ops id=\"CTIP-FFSA-FNMF\" doEntiteAff=\"true\" doGenerateB20=\"true\"> </ops>\n" +
            "    <ops id=\"POLEEMPLOI\" doEntiteAff=\"true\" doGenerateB20=\"true\"> </ops>\n" +
            "</listOps>";

    private static final XmldbURI LIST_OPS_XSLT_NAME = XmldbURI.create("testListOps.xsl");

    private static final String LIST_OPS_XSLT =
            "<xsl:stylesheet xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" xmlns:ts=\"http://www.talentia-software.fr\" version=\"2.0\">\n" +
            "    <xsl:output method=\"xml\" indent=\"no\" encoding=\"UTF-8\"/>\n" +
            "    <!-- -->\n" +
            "    <xsl:param name=\"listOpsFileUri\" required=\"yes\"/>\n" +
            "\n" +
            "    <!-- -->\n" +
            "    <xsl:variable name=\"ts:listOps\" select=\"doc($listOpsFileUri)\"/>\n" +
            "\n" +
            "    <xsl:key name=\"ts:listOpsById\" match=\"//ops\" use=\"@id\"/>\n" +
            "\n" +
            "    <!-- -->\n" +
            "    <xsl:template match=\"/\">\n" +
            "        <xsl:if test=\"empty($ts:listOps)\">\n" +
            "            <xsl:message terminate=\"yes\">Could not find listOpsFileUri document</xsl:message>\n" +
            "        </xsl:if>\n" +
            "\n" +
            "        <DSN_FLAT>\n" +
            "            <xsl:for-each select=\"//ops\">\n" +
            "                <xsl:variable name=\"keyId\" select=\"@id\"/>\n" +
            "                <xsl:variable name=\"refListOpsEntry\" select=\"$ts:listOps/key('ts:listOpsById', $keyId)\"/>\n" +
            "                <xsl:element name=\"keyId\">\n" +
            "                    <xsl:value-of select=\"$keyId\"/>\n" +
            "                </xsl:element>\n" +
            "                <xsl:element name=\"listOpsEntry\">\n" +
            "                    <xsl:for-each select=\"$refListOpsEntry/@*\">\n" +
            "                        <xsl:value-of select=\"concat(name(), ': ', ., ' ')\"/>\n" +
            "                    </xsl:for-each>\n" +
            "                </xsl:element>\n" +
            "            </xsl:for-each>\n" +
            "        </DSN_FLAT>\n" +
            "    </xsl:template>\n" +
            "</xsl:stylesheet>";

    private static final String LIST_OPS_XQUERY =
            "xquery version \"3.0\";\n" +
            "\n" +
            "(:Read document with xsl:for-each and look for key in the dictionary document :)\n" +
            "declare variable $xsltPath as xs:string := '" + TEST_IDS_COLLECTION.getCollectionPath() + "';\n" +
            "declare variable $listOpsFileUri as xs:string := '" + TEST_IDS_COLLECTION.getCollectionPath()+ "/listOpsErr.xml';\n" +
            "declare variable $inputFileUri as xs:string := '" + TEST_IDS_COLLECTION.getCollectionPath() + "/inputListOps.xml';\n" +
            "\n" +
            "let $params :=  <parameters>\n" +
            "                    <param name=\"listOpsFileUri\" value=\"{$listOpsFileUri}\" />\n" +
            "                </parameters>\n" +
            "\n" +
            "let $xmlData := doc($inputFileUri)\n" +
            "\n" +
            "return transform:transform($xmlData, doc(concat($xsltPath, '/', 'testListOps.xsl')),$params)";

    private static final XmldbURI TEST_DOCUMENT_XSLT_COLLECTION = XmldbURI.create("/db/transform-doc-test");
    private static final XmldbURI DOCUMENT_XSLT_NAME = XmldbURI.create("xsl-doc.xslt");

    private static final String DOCUMENT_XSLT =
            "<xsl:stylesheet xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" version=\"2.0\">\n" +
            "\t<xsl:template name=\"xsl-doc\">\n" +
            "\t\t<xsl:document><elem1/></xsl:document>\n" +
            "\t</xsl:template>\n" +
            "</xsl:stylesheet>";

    private static final String DOCUMENT_XSLT_QUERY =
            "import module namespace transform=\"http://exist-db.org/xquery/transform\";\n" +
            "\n" +
            "let $xsl := doc('" + TEST_DOCUMENT_XSLT_COLLECTION.append(DOCUMENT_XSLT_NAME).getRawCollectionPath() + "')\n" +
            "return\n" +
            "\ttransform:transform((), $xsl, (), <attributes><attr name=\"http://saxon.sf.net/feature/initialTemplate\" value=\"xsl-doc\"/></attributes>, ())";

    private static final XmldbURI SIMPLE_XML_NAME = XmldbURI.create("simple.xml");
    private static final XmldbURI TEST_SIMPLE_XML_COLLECTION = XmldbURI.create("/db/transform-simple-test");
    private static final XmldbURI TEST_SIMPLE_XML_WITH_COMMENT_COLLECTION = XmldbURI.create("/db/transform-simple-1c-test");
    private static final XmldbURI TEST_SIMPLE_XML_WITH_TWO_COMMENTS_COLLECTION = XmldbURI.create("/db/transform-simple-2c-test");

    private static final String SIMPLE_XML =
            "<n/>";

    private static final String SIMPLE_XML_WITH_COMMENT =
            "<!-- -->\n" +
            "<n/>";

    private static final String SIMPLE_XML_WITH_TWO_COMMENTS =
            "<!-- 1 --><!-- 2 -->\n" +
            "<n/>";

    private static final XmldbURI COUNT_DESCENDANTS_XSLT_NAME = XmldbURI.create("count-descendants.xslt");

    private static final XmldbURI TWO_NODES_XML_NAME = XmldbURI.create("two-nodes.xml");
    private static final XmldbURI TEST_TWO_NODES_COLLECTION = XmldbURI.create("/db/transform-two-nodes-test");
    private static final String TWO_NODES_XML = "<a><b/></a>";

    private static final XmldbURI COUNT_DESCENDANTS_TWO_NODES_XSLT_NAME = XmldbURI.create("count-descendants-two-nodes.xslt");

    private static final String COUNT_DESCENDANTS_TWO_NODES_XSLT =
            "<xsl:stylesheet xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" version=\"2.0\">\n" +
            "\n" +
            "\t<xsl:variable name=\"xml\" select=\"document('" + TEST_TWO_NODES_COLLECTION.append(TWO_NODES_XML_NAME).getRawCollectionPath() + "')\"/>\n" +
            "\t\n" +
            "\t<xsl:template match=\"/\">\n" +
            "\t\t<counts>\n" +
            "\t\t\t<count1><xsl:value-of select=\"count(doc($xml)//*)\"/></count1>\n" +
            "\t\t\t<count2><xsl:value-of select=\"count(doc($xml)/a//*)\"/></count2>\n" +
            "\t\t</counts>\n" +
            "\t</xsl:template>\n" +
            "\n" +
            "</xsl:stylesheet>";

    private static final String COUNT_DESCENDANTS_TWO_NODES_QUERY =
            "import module namespace transform=\"http://exist-db.org/xquery/transform\";\n" +
                "\n" +
                "let $xml := doc('" + TEST_TWO_NODES_COLLECTION.append(TWO_NODES_XML_NAME).getRawCollectionPath() + "')\n" +
                "let $xsl := doc('" + TEST_TWO_NODES_COLLECTION.append(COUNT_DESCENDANTS_TWO_NODES_XSLT_NAME).getRawCollectionPath() + "')\n" +
                "return\n" +
                "\ttransform:transform($xml, $xsl, ())";


    /**
     * {@see https://github.com/eXist-db/exist/issues/1506}
     */
    @Test
    public void keys() throws EXistException, PermissionDeniedException, XPathException {
        final BrokerPool pool = existEmbeddedServer.getBrokerPool();
        final XQuery xquery = pool.getXQueryService();
        try(final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject()))) {<commit_msg>[bugfix] Fix broken test<commit_after>/*
 * eXist Open Source Native XML Database
 * Copyright (C) 2001-2017 The eXist Project
 * http://exist-db.org
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
package org.exist.xquery.functions.transform;

import com.evolvedbinary.j8fu.tuple.Tuple2;
import org.exist.EXistException;
import org.exist.collections.Collection;
import org.exist.collections.IndexInfo;
import org.exist.collections.triggers.TriggerException;
import org.exist.security.PermissionDeniedException;
import org.exist.storage.BrokerPool;
import org.exist.storage.DBBroker;
import org.exist.storage.lock.Lock;
import org.exist.storage.lock.ManagedCollectionLock;
import org.exist.storage.txn.Txn;
import org.exist.test.ExistEmbeddedServer;
import org.exist.util.LockException;
import org.exist.xmldb.XmldbURI;
import org.exist.xquery.XPathException;
import org.exist.xquery.XQuery;
import org.exist.xquery.value.Item;
import org.exist.xquery.value.Sequence;
import org.exist.xquery.value.Type;
import org.junit.*;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import org.xmlunit.builder.DiffBuilder;
import org.xmlunit.builder.Input;
import org.xmlunit.diff.Diff;

import javax.xml.transform.Source;
import java.io.IOException;
import java.util.Optional;

import static com.evolvedbinary.j8fu.tuple.Tuple.Tuple;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNotNull;

/**
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 */
public class TransformTest {

    @ClassRule
    public static ExistEmbeddedServer existEmbeddedServer = new ExistEmbeddedServer(true, true);

    private static final XmldbURI TEST_IDS_COLLECTION = XmldbURI.create("/db/transform-ids-test");

    private static final XmldbURI INPUT_LIST_XML_NAME = XmldbURI.create("inputListOps.xml");

    private static final String INPUT_XML =
            "<listOps>\n" +
            "    <ops id=\"IRCANTEC\"/>\n" +
            "    <ops id=\"CIBTP\"/>\n" +
            "    <ops id=\"AGIRC-ARRCO\"/>\n" +
            "    <ops id=\"CTIP-FFSA-FNMF\"/>\n" +
            "</listOps>";

    private static final XmldbURI DICTIONARY_XML_NAME = XmldbURI.create("listOpsErr.xml");

    private static final String DICTIONARY_XML =
            "<listOps>\n" +
            "    <ops id=\"IRCANTEC\" doEntiteAff=\"false\" doGenerateB20=\"true\"> </ops>\n" +
            "    <ops id=\"CIBTP\" doEntiteAff=\"true\" doGenerateB20=\"true\"/>\n" +
            "    <ops id=\"AGIRC-ARRCO\" doEntiteAff=\"true\" doGenerateB20=\"false\"> </ops>\n" +
            "    <ops id=\"CTIP-FFSA-FNMF\" doEntiteAff=\"true\" doGenerateB20=\"true\"> </ops>\n" +
            "    <ops id=\"POLEEMPLOI\" doEntiteAff=\"true\" doGenerateB20=\"true\"> </ops>\n" +
            "</listOps>";

    private static final XmldbURI LIST_OPS_XSLT_NAME = XmldbURI.create("testListOps.xsl");

    private static final String LIST_OPS_XSLT =
            "<xsl:stylesheet xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" xmlns:ts=\"http://www.talentia-software.fr\" version=\"2.0\">\n" +
            "    <xsl:output method=\"xml\" indent=\"no\" encoding=\"UTF-8\"/>\n" +
            "    <!-- -->\n" +
            "    <xsl:param name=\"listOpsFileUri\" required=\"yes\"/>\n" +
            "\n" +
            "    <!-- -->\n" +
            "    <xsl:variable name=\"ts:listOps\" select=\"doc($listOpsFileUri)\"/>\n" +
            "\n" +
            "    <xsl:key name=\"ts:listOpsById\" match=\"//ops\" use=\"@id\"/>\n" +
            "\n" +
            "    <!-- -->\n" +
            "    <xsl:template match=\"/\">\n" +
            "        <xsl:if test=\"empty($ts:listOps)\">\n" +
            "            <xsl:message terminate=\"yes\">Could not find listOpsFileUri document</xsl:message>\n" +
            "        </xsl:if>\n" +
            "\n" +
            "        <DSN_FLAT>\n" +
            "            <xsl:for-each select=\"//ops\">\n" +
            "                <xsl:variable name=\"keyId\" select=\"@id\"/>\n" +
            "                <xsl:variable name=\"refListOpsEntry\" select=\"$ts:listOps/key('ts:listOpsById', $keyId)\"/>\n" +
            "                <xsl:element name=\"keyId\">\n" +
            "                    <xsl:value-of select=\"$keyId\"/>\n" +
            "                </xsl:element>\n" +
            "                <xsl:element name=\"listOpsEntry\">\n" +
            "                    <xsl:for-each select=\"$refListOpsEntry/@*\">\n" +
            "                        <xsl:value-of select=\"concat(name(), ': ', ., ' ')\"/>\n" +
            "                    </xsl:for-each>\n" +
            "                </xsl:element>\n" +
            "            </xsl:for-each>\n" +
            "        </DSN_FLAT>\n" +
            "    </xsl:template>\n" +
            "</xsl:stylesheet>";

    private static final String LIST_OPS_XQUERY =
            "xquery version \"3.0\";\n" +
            "\n" +
            "(:Read document with xsl:for-each and look for key in the dictionary document :)\n" +
            "declare variable $xsltPath as xs:string := '" + TEST_IDS_COLLECTION.getCollectionPath() + "';\n" +
            "declare variable $listOpsFileUri as xs:string := '" + TEST_IDS_COLLECTION.getCollectionPath()+ "/listOpsErr.xml';\n" +
            "declare variable $inputFileUri as xs:string := '" + TEST_IDS_COLLECTION.getCollectionPath() + "/inputListOps.xml';\n" +
            "\n" +
            "let $params :=  <parameters>\n" +
            "                    <param name=\"listOpsFileUri\" value=\"{$listOpsFileUri}\" />\n" +
            "                </parameters>\n" +
            "\n" +
            "let $xmlData := doc($inputFileUri)\n" +
            "\n" +
            "return transform:transform($xmlData, doc(concat($xsltPath, '/', 'testListOps.xsl')),$params)";

    private static final XmldbURI TEST_DOCUMENT_XSLT_COLLECTION = XmldbURI.create("/db/transform-doc-test");
    private static final XmldbURI DOCUMENT_XSLT_NAME = XmldbURI.create("xsl-doc.xslt");

    private static final String DOCUMENT_XSLT =
            "<xsl:stylesheet xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" version=\"2.0\">\n" +
            "\t<xsl:template name=\"xsl-doc\">\n" +
            "\t\t<xsl:document><elem1/></xsl:document>\n" +
            "\t</xsl:template>\n" +
            "</xsl:stylesheet>";

    private static final String DOCUMENT_XSLT_QUERY =
            "import module namespace transform=\"http://exist-db.org/xquery/transform\";\n" +
            "\n" +
            "let $xsl := doc('" + TEST_DOCUMENT_XSLT_COLLECTION.append(DOCUMENT_XSLT_NAME).getRawCollectionPath() + "')\n" +
            "return\n" +
            "\ttransform:transform((), $xsl, (), <attributes><attr name=\"http://saxon.sf.net/feature/initialTemplate\" value=\"xsl-doc\"/></attributes>, ())";

    private static final XmldbURI SIMPLE_XML_NAME = XmldbURI.create("simple.xml");
    private static final XmldbURI TEST_SIMPLE_XML_COLLECTION = XmldbURI.create("/db/transform-simple-test");
    private static final XmldbURI TEST_SIMPLE_XML_WITH_COMMENT_COLLECTION = XmldbURI.create("/db/transform-simple-1c-test");
    private static final XmldbURI TEST_SIMPLE_XML_WITH_TWO_COMMENTS_COLLECTION = XmldbURI.create("/db/transform-simple-2c-test");

    private static final String SIMPLE_XML =
            "<n/>";

    private static final String SIMPLE_XML_WITH_COMMENT =
            "<!-- -->\n" +
            "<n/>";

    private static final String SIMPLE_XML_WITH_TWO_COMMENTS =
            "<!-- 1 --><!-- 2 -->\n" +
            "<n/>";

    private static final XmldbURI COUNT_DESCENDANTS_XSLT_NAME = XmldbURI.create("count-descendants.xslt");

    private static final XmldbURI TWO_NODES_XML_NAME = XmldbURI.create("two-nodes.xml");
    private static final XmldbURI TEST_TWO_NODES_COLLECTION = XmldbURI.create("/db/transform-two-nodes-test");
    private static final String TWO_NODES_XML = "<a><b/></a>";

    private static final XmldbURI COUNT_DESCENDANTS_TWO_NODES_XSLT_NAME = XmldbURI.create("count-descendants-two-nodes.xslt");

    private static final String COUNT_DESCENDANTS_TWO_NODES_XSLT =
            "<xsl:stylesheet xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" version=\"2.0\">\n" +
            "\n" +
            "\t<xsl:variable name=\"xml\" select=\"doc('" + TEST_TWO_NODES_COLLECTION.append(TWO_NODES_XML_NAME).getRawCollectionPath() + "')\"/>\n" +
            "\t\n" +
            "\t<xsl:template match=\"/\">\n" +
            "\t\t<counts>\n" +
            "\t\t\t<count1><xsl:value-of select=\"count($xml//*)\"/></count1>\n" +
            "\t\t\t<count2><xsl:value-of select=\"count($xml/a//*)\"/></count2>\n" +
            "\t\t</counts>\n" +
            "\t</xsl:template>\n" +
            "\n" +
            "</xsl:stylesheet>";

    private static final String COUNT_DESCENDANTS_TWO_NODES_QUERY =
            "import module namespace transform=\"http://exist-db.org/xquery/transform\";\n" +
                "\n" +
                "let $xml := doc('" + TEST_TWO_NODES_COLLECTION.append(TWO_NODES_XML_NAME).getRawCollectionPath() + "')\n" +
                "let $xsl := doc('" + TEST_TWO_NODES_COLLECTION.append(COUNT_DESCENDANTS_TWO_NODES_XSLT_NAME).getRawCollectionPath() + "')\n" +
                "return\n" +
                "\ttransform:transform($xml, $xsl, ())";


    /**
     * {@see https://github.com/eXist-db/exist/issues/1506}
     */
    @Test
    public void keys() throws EXistException, PermissionDeniedException, XPathException {
        final BrokerPool pool = existEmbeddedServer.getBrokerPool();
        final XQuery xquery = pool.getXQueryService();
        try(final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject()))) {
{'fraction_non_alphanumeric': 0.16135102079217237, 'fraction_numerical': 0.006962084862169536, 'mean_word_length': 2.4488968202465933, 'pattern_counts': {'":': 0, '<': 173, '<?xml version=': 0, '>': 173, 'https://': 2, 'lorem ipsum': 0, 'www.': 8, 'xml': 48}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5530867', 'n_tokens_mistral': 7136, 'n_tokens_neox': 6836, 'n_words': 1235}