<commit_before>/*
 *  eXist Open Source Native XML Database
 *  Copyright (C) 2001-04,  The eXist Project
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version 2
 *  of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 * 
 *  $Id$
 */
package org.exist.storage;

import java.util.ArrayList;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import org.apache.log4j.Logger;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.exist.util.DatabaseConfigurationException;
import org.exist.dom.QName;


/**
 * Contains information about which parts of a document should be
 * fulltext-indexed for a specified doctype. It basically keeps a list of paths
 * to include and exclude from indexing. Paths are specified using
 * simple XPath syntax, e.g. //SPEECH will select any SPEECH elements,
 * //title/@id will select all id attributes being children of title elements.
 *
 * <pre>
 *      &lt;fulltext default="all" attributes="no">
 *          &lt;include path="//div/para"/>
 *          &lt;include path="//nested" content="mixed"/>
 *      &lt;/fulltext>
 * </pre>
 *
 * @author <NAME>
 */
public class FulltextIndexSpec {
	
    private static final String PATH_ATTRIB = "path";
    private static final String CONTENT_ATTRIB = "content";
    private static final String CONTENT_MIXED = "mixed";
    private static final String PRESERVE_CONTENT_ELEMENT = "preserveContent";
    private static final String EXCLUDE_INTERFACE = "exclude";
    private static final String INCLUDE_ELEMENT = "include";
    private static final String ALPHANUM_ATTRIB = "alphanum";
    private static final String ATTRIBUTES_ATTRIB = "attributes";
    private static final String DEFAULT_ATTRIB = "default";
    private static final String CREATE_ELEMENT = "create";
    private static final String QNAME_ATTRIB = "qname";

    private static final NodePath[] ARRAY_TYPE = new NodePath[0];
    
    private final static Logger LOG = Logger.getLogger(FulltextIndexSpec.class);
    
    protected NodePath[] includePath;
    protected NodePath[] excludePath;
    protected NodePath[] mixedPath;
    protected NodePath[] preserveContent;
    protected Set qnameSpecs = new TreeSet();

    protected boolean includeByDefault = true;
    protected boolean includeAttributes = true;
    protected boolean includeAlphaNum = true;

    /**
     * Constructor for the IndexPaths object
     *
     * param def if set to true, include everything by default. In this case
     * use exclude elements to specify the excluded parts.
     */
    public FulltextIndexSpec(Map namespaces, Element node) throws DatabaseConfigurationException {
        includeByDefault = true;
        ArrayList includeList = new ArrayList();
        ArrayList excludeList = new ArrayList();
        ArrayList preserveList = new ArrayList();
        ArrayList mixedList = new ArrayList();

        // check default settings
        String def = node.getAttribute(DEFAULT_ATTRIB);
        if(def != null && def.length() > 0) {
            includeByDefault = def.equals("all");
        }
        String indexAttributes = node.getAttribute(ATTRIBUTES_ATTRIB);
		if (indexAttributes != null && indexAttributes.length() > 0) {
			includeAttributes = indexAttributes.equals("true") || indexAttributes.equals("yes");
		}

		String indexAlphaNum = node.getAttribute(ALPHANUM_ATTRIB);
		if (indexAlphaNum != null && indexAlphaNum.length() > 0)
			setIncludeAlphaNum(indexAlphaNum.equals("true") || indexAlphaNum.equals("yes"));

		// check paths to include/exclude
		NodeList children = node.getChildNodes();
		String ps, content;
		Node next;
        Element elem;
        for(int j = 0; j < children.getLength(); j++) {
		    next = children.item(j);
		    if(INCLUDE_ELEMENT.equals(next.getLocalName())) {
                elem = (Element) next;
                content = elem.getAttribute(CONTENT_ATTRIB);
                ps = elem.getAttribute(PATH_ATTRIB);
                if (ps == null || ps.length() == 0) {
                    throw new DatabaseConfigurationException("include element requires an attribute 'path' in collection configuration.");
                }
                if (content != null && content.length() != 0 && CONTENT_MIXED.equals(content)) {
                    mixedList.add(new NodePath(namespaces, ps, false));
                } else {
                    includeList.add( new NodePath(namespaces, ps) );
                }
            } else if(EXCLUDE_INTERFACE.equals(next.getLocalName())) {
		        ps = ((Element) next).getAttribute(PATH_ATTRIB);
                excludeList.add( new NodePath(namespaces, ps) );
		    } else if(PRESERVE_CONTENT_ELEMENT.equals(next.getLocalName())) {
		        ps = ((Element) next).getAttribute(PATH_ATTRIB);
                preserveList.add( new NodePath(namespaces, ps) );
		    } else if(CREATE_ELEMENT.equals(next.getLocalName())) {
                elem = (Element) next;
                String name = elem.getAttribute(QNAME_ATTRIB);
                boolean isAttribute = false;
                if (name.startsWith("@")) {
                    isAttribute = true;
                    name = name.substring(1);<commit_msg>Throw exception for invalid collection configuration entry.<commit_after>/*
 *  eXist Open Source Native XML Database
 *  Copyright (C) 2001-04,  The eXist Project
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version 2
 *  of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 * 
 *  $Id$
 */
package org.exist.storage;

import java.util.ArrayList;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import org.apache.log4j.Logger;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.exist.util.DatabaseConfigurationException;
import org.exist.dom.QName;


/**
 * Contains information about which parts of a document should be
 * fulltext-indexed for a specified doctype. It basically keeps a list of paths
 * to include and exclude from indexing. Paths are specified using
 * simple XPath syntax, e.g. //SPEECH will select any SPEECH elements,
 * //title/@id will select all id attributes being children of title elements.
 *
 * <pre>
 *      &lt;fulltext default="all" attributes="no">
 *          &lt;include path="//div/para"/>
 *          &lt;include path="//nested" content="mixed"/>
 *      &lt;/fulltext>
 * </pre>
 *
 * @author <NAME>
 */
public class FulltextIndexSpec {
	
    private static final String PATH_ATTRIB = "path";
    private static final String CONTENT_ATTRIB = "content";
    private static final String CONTENT_MIXED = "mixed";
    private static final String PRESERVE_CONTENT_ELEMENT = "preserveContent";
    private static final String EXCLUDE_INTERFACE = "exclude";
    private static final String INCLUDE_ELEMENT = "include";
    private static final String ALPHANUM_ATTRIB = "alphanum";
    private static final String ATTRIBUTES_ATTRIB = "attributes";
    private static final String DEFAULT_ATTRIB = "default";
    private static final String CREATE_ELEMENT = "create";
    private static final String QNAME_ATTRIB = "qname";

    private static final NodePath[] ARRAY_TYPE = new NodePath[0];
    
    private final static Logger LOG = Logger.getLogger(FulltextIndexSpec.class);
    
    protected NodePath[] includePath;
    protected NodePath[] excludePath;
    protected NodePath[] mixedPath;
    protected NodePath[] preserveContent;
    protected Set qnameSpecs = new TreeSet();

    protected boolean includeByDefault = true;
    protected boolean includeAttributes = true;
    protected boolean includeAlphaNum = true;

    /**
     * Constructor for the IndexPaths object
     *
     * param def if set to true, include everything by default. In this case
     * use exclude elements to specify the excluded parts.
     */
    public FulltextIndexSpec(Map namespaces, Element node) throws DatabaseConfigurationException {
        includeByDefault = true;
        ArrayList includeList = new ArrayList();
        ArrayList excludeList = new ArrayList();
        ArrayList preserveList = new ArrayList();
        ArrayList mixedList = new ArrayList();

        // check default settings
        String def = node.getAttribute(DEFAULT_ATTRIB);
        if(def != null && def.length() > 0) {
            includeByDefault = def.equals("all");
        }
        String indexAttributes = node.getAttribute(ATTRIBUTES_ATTRIB);
		if (indexAttributes != null && indexAttributes.length() > 0) {
			includeAttributes = indexAttributes.equals("true") || indexAttributes.equals("yes");
		}

		String indexAlphaNum = node.getAttribute(ALPHANUM_ATTRIB);
		if (indexAlphaNum != null && indexAlphaNum.length() > 0)
			setIncludeAlphaNum(indexAlphaNum.equals("true") || indexAlphaNum.equals("yes"));

		// check paths to include/exclude
		NodeList children = node.getChildNodes();
		String ps, content;
		Node next;
        Element elem;
        for(int j = 0; j < children.getLength(); j++) {
		    next = children.item(j);
		    if(INCLUDE_ELEMENT.equals(next.getLocalName())) {
                elem = (Element) next;
                content = elem.getAttribute(CONTENT_ATTRIB);
                ps = elem.getAttribute(PATH_ATTRIB);
                if (ps == null || ps.length() == 0) {
                    throw new DatabaseConfigurationException("include element requires an attribute 'path' in collection configuration.");
                }
                if (content != null && content.length() != 0 && CONTENT_MIXED.equals(content)) {
                    mixedList.add(new NodePath(namespaces, ps, false));
                } else {
                    includeList.add( new NodePath(namespaces, ps) );
                }
            } else if(EXCLUDE_INTERFACE.equals(next.getLocalName())) {
		        ps = ((Element) next).getAttribute(PATH_ATTRIB);
                if (ps == null || ps.length() == 0) {
                    throw new DatabaseConfigurationException("exclude element requires an attribute 'path' in collection configuration.");
                }
                excludeList.add( new NodePath(namespaces, ps) );
            } else if(PRESERVE_CONTENT_ELEMENT.equals(next.getLocalName())) {
		        ps = ((Element) next).getAttribute(PATH_ATTRIB);
                if (ps == null || ps.length() == 0) {
                    throw new DatabaseConfigurationException("preserveContent element requires an attribute 'path' in collection configuration.");
                }
                preserveList.add( new NodePath(namespaces, ps) );
		    } else if(CREATE_ELEMENT.equals(next.getLocalName())) {
                elem = (Element) next;
                String name = elem.getAttribute(QNAME_ATTRIB);
                if (name == null || name.length() == 0) {
                    throw new DatabaseConfigurationException("create element requires an attribute 'qname' in collection configuration.");
                }
                boolean isAttribute = false;
                if (name.startsWith("@")) {
                    isAttribute = true;
                    name = name.substring(1);
{'fraction_non_alphanumeric': 0.09695493300852619, 'fraction_numerical': 0.005602923264311815, 'mean_word_length': 2.8741742686379363, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4617130', 'n_tokens_mistral': 3340, 'n_tokens_neox': 3185, 'n_words': 1065}