<commit_before>/*
 *  eXist Open Source Native XML Database
 *  Copyright (C) 2001-03 <NAME>
 *  <EMAIL>
 *  http://exist.sourceforge.net
 *  
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public License
 *  as published by the Free Software Foundation; either version 2
 *  of the License, or (at your option) any later version.
 *  
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *  
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *  
 *  $Id$
 */
package org.exist.cocoon;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;

import javax.servlet.http.HttpServletRequest;

import org.apache.avalon.framework.configuration.Configurable;
import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
import org.apache.avalon.framework.parameters.ParameterException;
import org.apache.avalon.framework.parameters.Parameterizable;
import org.apache.avalon.framework.parameters.Parameters;
import org.apache.cocoon.ProcessingException;
import org.apache.cocoon.caching.CacheableProcessingComponent;
import org.apache.cocoon.environment.Context;
import org.apache.cocoon.environment.ObjectModelHelper;
import org.apache.cocoon.environment.Request;
import org.apache.cocoon.environment.Response;
import org.apache.cocoon.environment.Session;
import org.apache.cocoon.environment.SourceResolver;
import org.apache.cocoon.environment.http.HttpEnvironment;
import org.apache.cocoon.generation.ServiceableGenerator;
import org.apache.cocoon.xml.IncludeXMLConsumer;
import org.apache.excalibur.source.Source;
import org.apache.excalibur.source.SourceValidity;
import org.apache.excalibur.source.impl.validity.AggregatedValidity;
import org.apache.excalibur.source.impl.validity.ExpiresValidity;
import org.exist.source.CocoonSource;
import org.exist.storage.serializers.EXistOutputKeys;
import org.exist.storage.serializers.Serializer;
import org.exist.xmldb.CollectionImpl;
import org.exist.xmldb.XQueryService;
import org.exist.xquery.XPathException;
import org.exist.xquery.functions.request.RequestModule;
import org.exist.xquery.value.Item;
import org.exist.xquery.value.Sequence;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xmldb.api.DatabaseManager;
import org.xmldb.api.base.Collection;
import org.xmldb.api.base.Database;
import org.xmldb.api.base.ResourceSet;
import org.xmldb.api.base.XMLDBException;
import org.xmldb.api.modules.XMLResource;

/**
 * A generator for Cocoon which reads an XQuery script, executes it and passes
 * the results into the Cocoon pipeline.
 * 
 * The following optional attributes are accepted on the component declaration as default eXist settings:
 * <li><tt>collection</tt>: identifies the XML:DB root collection used to process
 * the request</li>
 * <li><tt>user</tt></li>
 * <li><tt>password</tt></li>
 * <li><tt>create-session</tt>: if set to "true", indicates that an
 * HTTP session should be created upon the first invocation.</li>
 * <li><tt>expand-xincludes</tt></li>
 * <li><tt>cache-validity</tt>: if specified, the XQuery content is
 * cached until the specified delay expressed in milliseconds is elapsed
 * or until the XQuery file is modified.  The identity of the cached content is
 * computed using the XQuery file URI and the list of all parameters passed to
 * the XQuery.</li>
 * 
 * The component also accept default parameters that will be declared as implicit variables in the XQuery.
 * See below an example declaration of the XQueryGenerator component with default eXist settings, and an extra user-defined parameter:
 * 
 * <map:generator logger="xmldb" name="xquery"
 * 		collection="xmldb:exist:///db/"
 * 		user="guest"
 * 		password="<PASSWORD>"
 *		create-session="false"
 * 		expand-xincludes="false"
 *		cache-validity="-1"
 *		src="org.exist.cocoon.XQueryGenerator">
 *   <parameter name="myProjectURI" value="/db/myproject"/>
 * </map:generator>
 * 
 * These settings and parameters can be overriden on a per-pipeline basis with sitemap parameters, see below with default values and the extra user-defined parameter:
 * 
 * <pre>
 *  &lt;map:parameter name=&quot;collection&quot; value=&quot;xmldb:exist:///db&quot;/&gt;
 *  &lt;map:parameter name=&quot;user&quot; value=&quot;guest&quot;/&gt;
 *  &lt;map:parameter name=&quot;password&quot; value=&quot;guest&quot;/&gt;
 *  &lt;map:parameter name=&quot;create-session&quot; value=&quot;false&quot;/&gt;
 *  &lt;map:parameter name=&quot;expand-xincludes&quot; value=&quot;false&quot;/&gt;
 *  &lt;map:parameter name=&quot;cache-validity&quot; value=&quot;-1quot;/&gt;
 *  &lt;map:parameter name=&quot;myProjectURI&quot; value=&quot;/db/myproject&quot;/&gt;
 * </pre>
 * 
 * The last sitemap parameter overrides the value of the XQuery variable defined in the component parameters,
 * whereas others override the default eXist settings defined on the component attributes.
 *
 * @author wolf
 */
public class XQueryGenerator extends ServiceableGenerator implements Configurable, Parameterizable, CacheableProcessingComponent {
	public final static String DRIVER = "org.exist.xmldb.DatabaseImpl";

	private Source inputSource = null;
	private Map objectModel = null;

	private boolean createSession;
	private boolean defaultCreateSession = false;
	private final static String CREATE_SESSION = "create-session";

	private boolean expandXIncludes;
	private boolean defaultExpandXIncludes = false;
	private final static String EXPAND_XINCLUDES = "expand-xincludes";

	private String collectionURI;
	private String defaultCollectionURI = "xmldb:exist:///db";
	private final static String COLLECTION_URI = "collection";
	
	private long cacheValidity;
	private long defaultCacheValidity = -1;
	private final static String CACHE_VALIDITY = "cache-validity";

	private String user;
	private String defaultUser = "guest";
	private final static String USER = "user";

	private String password;
	private String defaultPassword = "<PASSWORD>";
	private final static String PASSWORD = "password";

	private Map optionalParameters;
	private Parameters componentParams;

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.apache.cocoon.generation.AbstractGenerator#setup(org.apache.cocoon.environment.SourceResolver,
	 *         java.util.Map, java.lang.String,
	 *         org.apache.avalon.framework.parameters.Parameters)
	 */
	public void setup(SourceResolver resolver, Map objectModel, String source,
						Parameters parameters) throws ProcessingException,
			SAXException, IOException {
		super.setup(resolver, objectModel, source, parameters);

		/*
		 * We don't do this directly in parameterize() because setup() can be
		 * called multiple times and optionalParameters needs resetting to forget
		 * sitemap parameters that may have been removed inbetween
		 */
		this.optionalParameters = new HashMap();
		String paramNames[] = componentParams.getNames();
		for (int i = 0; i < paramNames.length; i++) {
			String param = paramNames[i];
			try {
				optionalParameters.put(param, componentParams.getParameter(param));
			} catch (ParameterException e1) {
				// Cannot happen as we iterate through existing parameters
			}
		}
<commit_msg>Applied patch by <NAME>: XQueryGenerator parameters must be sorted for caching or caching will not work if parameters change position. getKey() should return a stable key.<commit_after>/*
 *  eXist Open Source Native XML Database
 *  Copyright (C) 2001-03 <NAME>
 *  <EMAIL>
 *  http://exist.sourceforge.net
 *  
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public License
 *  as published by the Free Software Foundation; either version 2
 *  of the License, or (at your option) any later version.
 *  
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *  
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *  
 *  $Id$
 */
package org.exist.cocoon;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;

import javax.servlet.http.HttpServletRequest;

import org.apache.avalon.framework.configuration.Configurable;
import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
import org.apache.avalon.framework.parameters.ParameterException;
import org.apache.avalon.framework.parameters.Parameterizable;
import org.apache.avalon.framework.parameters.Parameters;
import org.apache.cocoon.ProcessingException;
import org.apache.cocoon.caching.CacheableProcessingComponent;
import org.apache.cocoon.environment.Context;
import org.apache.cocoon.environment.ObjectModelHelper;
import org.apache.cocoon.environment.Request;
import org.apache.cocoon.environment.Response;
import org.apache.cocoon.environment.Session;
import org.apache.cocoon.environment.SourceResolver;
import org.apache.cocoon.environment.http.HttpEnvironment;
import org.apache.cocoon.generation.ServiceableGenerator;
import org.apache.cocoon.xml.IncludeXMLConsumer;
import org.apache.excalibur.source.Source;
import org.apache.excalibur.source.SourceValidity;
import org.apache.excalibur.source.impl.validity.AggregatedValidity;
import org.apache.excalibur.source.impl.validity.ExpiresValidity;
import org.exist.source.CocoonSource;
import org.exist.storage.serializers.EXistOutputKeys;
import org.exist.storage.serializers.Serializer;
import org.exist.xmldb.CollectionImpl;
import org.exist.xmldb.XQueryService;
import org.exist.xquery.XPathException;
import org.exist.xquery.functions.request.RequestModule;
import org.exist.xquery.value.Sequence;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xmldb.api.DatabaseManager;
import org.xmldb.api.base.Collection;
import org.xmldb.api.base.Database;
import org.xmldb.api.base.ResourceSet;
import org.xmldb.api.base.XMLDBException;
import org.xmldb.api.modules.XMLResource;

/**
 * A generator for Cocoon which reads an XQuery script, executes it and passes
 * the results into the Cocoon pipeline.
 * 
 * The following optional attributes are accepted on the component declaration as default eXist settings:
 * <li><tt>collection</tt>: identifies the XML:DB root collection used to process
 * the request</li>
 * <li><tt>user</tt></li>
 * <li><tt>password</tt></li>
 * <li><tt>create-session</tt>: if set to "true", indicates that an
 * HTTP session should be created upon the first invocation.</li>
 * <li><tt>expand-xincludes</tt></li>
 * <li><tt>cache-validity</tt>: if specified, the XQuery content is
 * cached until the specified delay expressed in milliseconds is elapsed
 * or until the XQuery file is modified.  The identity of the cached content is
 * computed using the XQuery file URI and the list of all parameters passed to
 * the XQuery.</li>
 * 
 * The component also accept default parameters that will be declared as implicit variables in the XQuery.
 * See below an example declaration of the XQueryGenerator component with default eXist settings, and an extra user-defined parameter:
 * 
 * <map:generator logger="xmldb" name="xquery"
 * 		collection="xmldb:exist:///db/"
 * 		user="guest"
 * 		password="<PASSWORD>"
 *		create-session="false"
 * 		expand-xincludes="false"
 *		cache-validity="-1"
 *		src="org.exist.cocoon.XQueryGenerator">
 *   <parameter name="myProjectURI" value="/db/myproject"/>
 * </map:generator>
 * 
 * These settings and parameters can be overriden on a per-pipeline basis with sitemap parameters, see below with default values and the extra user-defined parameter:
 * 
 * <pre>
 *  &lt;map:parameter name=&quot;collection&quot; value=&quot;xmldb:exist:///db&quot;/&gt;
 *  &lt;map:parameter name=&quot;user&quot; value=&quot;guest&quot;/&gt;
 *  &lt;map:parameter name=&quot;password&quot; value=&quot;guest&quot;/&gt;
 *  &lt;map:parameter name=&quot;create-session&quot; value=&quot;false&quot;/&gt;
 *  &lt;map:parameter name=&quot;expand-xincludes&quot; value=&quot;false&quot;/&gt;
 *  &lt;map:parameter name=&quot;cache-validity&quot; value=&quot;-1quot;/&gt;
 *  &lt;map:parameter name=&quot;myProjectURI&quot; value=&quot;/db/myproject&quot;/&gt;
 * </pre>
 * 
 * The last sitemap parameter overrides the value of the XQuery variable defined in the component parameters,
 * whereas others override the default eXist settings defined on the component attributes.
 *
 * @author wolf
 */
public class XQueryGenerator extends ServiceableGenerator implements Configurable, Parameterizable, CacheableProcessingComponent {
	public final static String DRIVER = "org.exist.xmldb.DatabaseImpl";

	private Source inputSource = null;
	private Map objectModel = null;

	private boolean createSession;
	private boolean defaultCreateSession = false;
	private final static String CREATE_SESSION = "create-session";

	private boolean expandXIncludes;
	private boolean defaultExpandXIncludes = false;
	private final static String EXPAND_XINCLUDES = "expand-xincludes";

	private String collectionURI;
	private String defaultCollectionURI = "xmldb:exist:///db";
	private final static String COLLECTION_URI = "collection";
	
	private long cacheValidity;
	private long defaultCacheValidity = -1;
	private final static String CACHE_VALIDITY = "cache-validity";

	private String user;
	private String defaultUser = "guest";
	private final static String USER = "user";

	private String password;
	private String defaultPassword = "<PASSWORD>";
	private final static String PASSWORD = "password";

	private Map optionalParameters;
	private Parameters componentParams;

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.apache.cocoon.generation.AbstractGenerator#setup(org.apache.cocoon.environment.SourceResolver,
	 *         java.util.Map, java.lang.String,
	 *         org.apache.avalon.framework.parameters.Parameters)
	 */
	public void setup(SourceResolver resolver, Map objectModel, String source,
						Parameters parameters) throws ProcessingException,
			SAXException, IOException {
		super.setup(resolver, objectModel, source, parameters);

		/*
		 * We don't do this directly in parameterize() because setup() can be
		 * called multiple times and optionalParameters needs resetting to forget
		 * sitemap parameters that may have been removed inbetween
		 * 
		 * The map must be sorted so that getKey() always returns the same
		 * object for any given oder of parameters.
		 */
		this.optionalParameters = new TreeMap();
		String paramNames[] = componentParams.getNames();
		for (int i = 0; i < paramNames.length; i++) {
			String param = paramNames[i];
			try {
				optionalParameters.put(param, componentParams.getParameter(param));
			} catch (ParameterException e1) {
				// Cannot happen as we iterate through existing parameters
			}
		}

{'fraction_non_alphanumeric': 0.10129820298011127, 'fraction_numerical': 0.002558035428790689, 'mean_word_length': 4.966425028615032, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 32}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3736162', 'n_tokens_mistral': 4686, 'n_tokens_neox': 4433, 'n_words': 1334}