<commit_before>/*
 * This library is part of OpenCms -
 * the Open Source Content Management System
 *
 * Copyright (c) Alkacon Software GmbH (http://www.alkacon.com)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * For further information about Alkacon Software GmbH, please see the
 * company website: http://www.alkacon.com
 *
 * For further information about OpenCms, please see the
 * project website: http://www.opencms.org
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

package org.opencms.search;

import org.opencms.db.CmsDriverManager;
import org.opencms.db.CmsPublishedResource;
import org.opencms.file.CmsObject;
import org.opencms.file.CmsResource;
import org.opencms.file.CmsResourceFilter;
import org.opencms.i18n.CmsMessageContainer;
import org.opencms.loader.CmsLoaderException;
import org.opencms.main.CmsEvent;
import org.opencms.main.CmsException;
import org.opencms.main.CmsIllegalArgumentException;
import org.opencms.main.CmsIllegalStateException;
import org.opencms.main.CmsLog;
import org.opencms.main.I_CmsEventListener;
import org.opencms.main.OpenCms;
import org.opencms.report.CmsLogReport;
import org.opencms.report.I_CmsReport;
import org.opencms.scheduler.I_CmsScheduledJob;
import org.opencms.search.documents.A_CmsVfsDocument;
import org.opencms.search.documents.CmsExtractionResultCache;
import org.opencms.search.documents.I_CmsDocumentFactory;
import org.opencms.search.documents.I_CmsTermHighlighter;
import org.opencms.search.fields.A_CmsSearchFieldConfiguration;
import org.opencms.search.fields.CmsSearchField;
import org.opencms.search.fields.CmsSearchFieldConfiguration;
import org.opencms.search.fields.CmsSearchFieldMapping;
import org.opencms.search.fields.I_CmsSearchField;
import org.opencms.search.fields.I_CmsSearchFieldConfiguration;
import org.opencms.search.galleries.CmsGallerySearchAnalyzer;
import org.opencms.search.solr.CmsSolrConfiguration;
import org.opencms.search.solr.CmsSolrFieldConfiguration;
import org.opencms.search.solr.CmsSolrIndex;
import org.opencms.security.CmsRole;
import org.opencms.security.CmsRoleViolationException;
import org.opencms.util.A_CmsModeStringEnumeration;
import org.opencms.util.CmsStringUtil;
import org.opencms.util.CmsUUID;

import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;

import org.apache.commons.logging.Log;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.Version;

/**
 * Implements the general management and configuration of the search and 
 * indexing facilities in OpenCms.<p>
 * 
 * @since 6.0.0 
 */
public class CmsSearchManager implements I_CmsScheduledJob, I_CmsEventListener {

    /**
     *  Enumeration class for force unlock types.<p>
     */
    public static final class CmsSearchForceUnlockMode extends A_CmsModeStringEnumeration {

        /** Force unlock type "always". */
        public static final CmsSearchForceUnlockMode ALWAYS = new CmsSearchForceUnlockMode("always");

        /** Force unlock type "never". */
        public static final CmsSearchForceUnlockMode NEVER = new CmsSearchForceUnlockMode("never");

        /** Force unlock type "only full". */
        public static final CmsSearchForceUnlockMode ONLYFULL = new CmsSearchForceUnlockMode("onlyfull");

        /** Serializable version id. */
        private static final long serialVersionUID = 74746076708908673L;

        /**
         * Creates a new force unlock type with the given name.<p>
         * 
         * @param mode the mode id to use
         */
        protected CmsSearchForceUnlockMode(String mode) {

            super(mode);
        }

        /**
         * Returns the lock type for the given type value.<p>
         *  
         * @param type the type value to get the lock type for
         * 
         * @return the lock type for the given type value
         */
        public static CmsSearchForceUnlockMode valueOf(String type) {

            if (type.equals(ALWAYS.toString())) {
                return ALWAYS;
            } else if (type.equals(NEVER.toString())) {
                return NEVER;
            } else {
                return ONLYFULL;
            }
        }
    }

    /**
     * Handles offline index generation.<p>
     */
    protected class CmsSearchOfflineHandler implements I_CmsEventListener {

        /** Indicates if the event handlers for the offline search have been already registered. */
        private boolean m_isEventRegistered;

        /** The list of resources to index. */
        private List<CmsPublishedResource> m_resourcesToIndex;

        /**
         * Initializes the offline index handler.<p>
         */
        protected CmsSearchOfflineHandler() {

            m_resourcesToIndex = new ArrayList<CmsPublishedResource>();
        }

        /**
         * Implements the event listener of this class.<p>
         * 
         * @see org.opencms.main.I_CmsEventListener#cmsEvent(org.opencms.main.CmsEvent)
         */
        @SuppressWarnings("unchecked")
        public void cmsEvent(CmsEvent event) {

            switch (event.getType()) {
                case I_CmsEventListener.EVENT_PROPERTY_MODIFIED:
                case I_CmsEventListener.EVENT_RESOURCE_CREATED:
                case I_CmsEventListener.EVENT_RESOURCE_AND_PROPERTIES_MODIFIED:
                case I_CmsEventListener.EVENT_RESOURCE_MODIFIED:
                    Object change = event.getData().get(I_CmsEventListener.KEY_CHANGE);
                    if ((change != null) && change.equals(new Integer(CmsDriverManager.NOTHING_CHANGED))) {
                        // skip lock & unlock
                        return;
                    }
                    // a resource has been modified - offline indexes require (re)indexing
                    List<CmsResource> resources = Collections.singletonList((CmsResource)event.getData().get(
                        I_CmsEventListener.KEY_RESOURCE));
                    reIndexResources(resources);
                    break;
                case I_CmsEventListener.EVENT_RESOURCES_AND_PROPERTIES_MODIFIED:
                case I_CmsEventListener.EVENT_RESOURCE_MOVED:
                case I_CmsEventListener.EVENT_RESOURCE_DELETED:
                case I_CmsEventListener.EVENT_RESOURCE_COPIED:
                case I_CmsEventListener.EVENT_RESOURCES_MODIFIED:
                    // a list of resources has been modified - offline indexes require (re)indexing
                    reIndexResources((List<CmsResource>)event.getData().get(I_CmsEventListener.KEY_RESOURCES));
                    break;
                default:
                    // no operation
            }
        }

        /**
         * Adds a list of {@link CmsPublishedResource} objects to be indexed.<p>
         * 
         * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to be indexed
         */
        protected synchronized void addResourcesToIndex(List<CmsPublishedResource> resourcesToIndex) {

            m_resourcesToIndex.addAll(resourcesToIndex);
        }

        /**
         * Returns the list of {@link CmsPublishedResource} objects to index.<p>
         *
         * @return the resources to index
         */
        protected synchronized List<CmsPublishedResource> getResourcesToIndex() {

            List<CmsPublishedResource> result = m_resourcesToIndex;
            m_resourcesToIndex = new ArrayList<CmsPublishedResource>();
            return result;
        }

        /**
         * Initializes this offline search handler, registering the event handlers if required.<p>
         */
        protected void initialize() {

            if (m_offlineIndexes.size() > 0) {
                // there is at least one offline index configured
                if ((m_offlineIndexThread == null) || !m_offlineIndexThread.isAlive()) {
                    // create the offline indexing thread
                    m_offlineIndexThread = new CmsSearchOfflineIndexThread(this);
                    // start the offline index thread
                    m_offlineIndexThread.start();
                }
            } else {
                if ((m_offlineIndexThread != null) && m_offlineIndexThread.isAlive()) {
                    // no offline indexes but thread still running, stop the thread
                    m_offlineIndexThread.shutDown();
                    m_offlineIndexThread = null;
                }
            }
            // do this only in case there are offline indexes configured
            if (!m_isEventRegistered && (m_offlineIndexes.size() > 0)) {
                m_isEventRegistered = true;
                // register this object as event listener
                OpenCms.addCmsEventListener(this, new int[] {
                    I_CmsEventListener.EVENT_PROPERTY_MODIFIED,
                    I_CmsEventListener.EVENT_RESOURCE_CREATED,
                    I_CmsEventListener.EVENT_RESOURCE_AND_PROPERTIES_MODIFIED,
                    I_CmsEventListener.EVENT_RESOURCE_MODIFIED,
                    I_CmsEventListener.EVENT_RESOURCES_AND_PROPERTIES_MODIFIED,
                    I_CmsEventListener.EVENT_RESOURCE_MOVED,
                    I_CmsEventListener.EVENT_RESOURCE_DELETED,
                    I_CmsEventListener.EVENT_RESOURCE_COPIED,
                    I_CmsEventListener.EVENT_RESOURCES_MODIFIED});
            }
        }

        /**
         * Updates all offline indexes for the given list of {@link CmsResource} objects.<p>
         * 
         * @param resources a list of {@link CmsResource} objects to update in the offline indexes
         */
        protected synchronized void reIndexResources(List<CmsResource> resources) {

            List<CmsPublishedResource> resourcesToIndex = new ArrayList<CmsPublishedResource>(resources.size());
            for (CmsResource res : resources) {
                CmsPublishedResource pubRes = new CmsPublishedResource(res);
                resourcesToIndex.add(pubRes);
            }
            if (resourcesToIndex.size() > 0) {
                // add the resources found to the offline index thread
                addResourcesToIndex(resourcesToIndex);
            }
        }
    }

    /**
     * The offline indexer thread runs periodically and indexes all resources added by the event handler.<p>
     */
    protected class CmsSearchOfflineIndexThread extends Thread {

        /** The event handler that triggers this thread. */
        CmsSearchOfflineHandler m_handler;

        /** Indicates if this thread is still alive. */
        boolean m_isAlive;

        /** If true a manual update (after file upload) was triggered. */
        private boolean m_updateTriggered;

        /**
         * Constructor.<p>
         * 
         * @param handler the offline index event handler
         */
        protected CmsSearchOfflineIndexThread(CmsSearchOfflineHandler handler) {

            super("OpenCms: Offline Search Indexer");
            m_handler = handler;
        }

        /**
         * @see java.lang.Thread#interrupt()
         */
        @Override
        public void interrupt() {

            super.interrupt();
            m_updateTriggered = true;
        }

        /**
         * @see java.lang.Thread#run()
         */
        @Override
        public void run() {

            // create a log report for the output
            I_CmsReport report = new CmsLogReport(m_adminCms.getRequestContext().getLocale(), CmsSearchManager.class);
            long offlineUpdateFrequency = getOfflineUpdateFrequency();
            m_updateTriggered = false;
            try {
                while (m_isAlive) {
                    if (!m_updateTriggered) {
                        try {
                            sleep(offlineUpdateFrequency);
                        } catch (InterruptedException e) {
                            // continue the thread after interruption
                            if (!m_isAlive) {
                                // the thread has been shut down while sleeping
                                continue;
                            }
                            if (offlineUpdateFrequency != getOfflineUpdateFrequency()) {
                                // offline update frequency change - clear interrupt status
                                offlineUpdateFrequency = getOfflineUpdateFrequency();
                            }
                        }
                    }
                    if (m_isAlive) {
                        // set update trigger to false since we do the update now
                        m_updateTriggered = false;
                        // get list of resource to update
                        List<CmsPublishedResource> resourcesToIndex = getResourcesToIndex();
                        if (resourcesToIndex.size() > 0) {
                            // only start indexing if there is at least one resource
                            startOfflineUpdateThread(report, resourcesToIndex);
                        }
                        // this is just called to clear the interrupt status of the thread
                        interrupted();
                    }
                }
            } finally {
                // make sure that live status is reset in case of Exceptions
                m_isAlive = false;
            }

        }

        /**
         * @see java.lang.Thread#start()
         */
        @Override
        public synchronized void start() {

            m_isAlive = true;
            super.start();
        }

        /**
         * Obtains the list of resource to update in the offline index, 
         * then optimizes the list by removing duplicate entries.<p>
         * 
         * @return the list of resource to update in the offline index
         */
        protected List<CmsPublishedResource> getResourcesToIndex() {

            List<CmsPublishedResource> resourcesToIndex = m_handler.getResourcesToIndex();
            List<CmsPublishedResource> result = new ArrayList<CmsPublishedResource>(resourcesToIndex.size());

            for (CmsPublishedResource pubRes : resourcesToIndex) {
                int pos = result.indexOf(pubRes);
                if (pos < 0) {
                    // resource not already contained in the update list
                    result.add(pubRes);
                } else {
                    CmsPublishedResource curRes = result.get(pos);
                    if ((pubRes.getState() != curRes.getState())
                        || (pubRes.getMovedState() != curRes.getMovedState())
                        || !pubRes.getRootPath().equals(curRes.getRootPath())) {
                        // resource already in the update list but new state is different, so also add this
                        result.add(pubRes);
                    }
                }
            }

            return result;
        }

        /**
         * Shuts down this offline index thread.<p>
         */
        protected void shutDown() {

            m_isAlive = false;
            interrupt();
        }

        /**
         * Updates the offline search indexes for the given list of resources.<p>
         * 
         * @param report the report to write the index information to
         * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to index
         */
        protected void startOfflineUpdateThread(I_CmsReport report, List<CmsPublishedResource> resourcesToIndex) {

            CmsSearchOfflineIndexWorkThread thread = new CmsSearchOfflineIndexWorkThread(report, resourcesToIndex);
            long startTime = System.currentTimeMillis();
            long waitTime = getOfflineUpdateFrequency() / 2;
            if (LOG.isDebugEnabled()) {
                LOG.debug(Messages.get().getBundle().key(
                    Messages.LOG_OI_UPDATE_START_1,
                    Integer.valueOf(resourcesToIndex.size())));
            }
            thread.start();

            do {
                try {
                    // wait half the time of the offline index frequency for the thread to finish
                    thread.join(waitTime);
                } catch (InterruptedException e) {
                    // continue
                }
                if (thread.isAlive()) {
                    LOG.warn(Messages.get().getBundle().key(
                        Messages.LOG_OI_UPDATE_LONG_2,
                        Integer.valueOf(resourcesToIndex.size()),
                        Long.valueOf(System.currentTimeMillis() - startTime)));
                }
            } while (thread.isAlive());

            if (LOG.isDebugEnabled()) {
                LOG.debug(Messages.get().getBundle().key(
                    Messages.LOG_OI_UPDATE_FINISH_2,
                    Integer.valueOf(resourcesToIndex.size()),
                    Long.valueOf(System.currentTimeMillis() - startTime)));
            }
        }
    }

    /**
     * An offline index worker Thread runs each time for every offline index update action.<p>
     * 
     * This was decoupled from the main {@link CmsSearchOfflineIndexThread} in order to avoid 
     * problems if a single operation "hangs" the Tread.<p>
     */
    protected class CmsSearchOfflineIndexWorkThread extends Thread {

        /** The report to write the index information to. */
        I_CmsReport m_report;

        /** The list of {@link CmsPublishedResource} objects to index. */
        List<CmsPublishedResource> m_resourcesToIndex;

        /**
         * Updates the offline search indexes for the given list of resources.<p>
         * 
         * @param report the report to write the index information to
         * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to index
         */
        protected CmsSearchOfflineIndexWorkThread(I_CmsReport report, List<CmsPublishedResource> resourcesToIndex) {

            super("OpenCms: Offline Search Index Worker");
            m_report = report;
            m_resourcesToIndex = resourcesToIndex;
        }

        /**
         * @see java.lang.Thread#run()
         */
        @Override
        public void run() {

            updateIndexOffline(m_report, m_resourcesToIndex);
        }
    }

    /** The default value used for generating search result excerpts (1024 chars). */
    public static final int DEFAULT_EXCERPT_LENGTH = 1024;

    /** The default value used for keeping the extraction results in the cache (672 hours = 4 weeks). */
    public static final float DEFAULT_EXTRACTION_CACHE_MAX_AGE = 672.0f;

    /** Default for the maximum number of modifications before a commit in the search index is triggered (500). */
    public static final int DEFAULT_MAX_MODIFICATIONS_BEFORE_COMMIT = 500;

    /** The default update frequency for offline indexes (15000 msec = 15 sec). */
    public static final int DEFAULT_OFFLINE_UPDATE_FREQNENCY = 15000;

    /** The default timeout value used for generating a document for the search index (60000 msec = 1 min). */
    public static final int DEFAULT_TIMEOUT = 60000;

    /** Scheduler parameter: Update only a specified list of indexes. */
    public static final String JOB_PARAM_INDEXLIST = "indexList";

    /** Scheduler parameter: Write the output of the update to the logfile. */
    public static final String JOB_PARAM_WRITELOG = "writeLog";

    /** Prefix for Lucene default analyzers package (<code>org.apache.lucene.analysis.</code>). */
    public static final String LUCENE_ANALYZER = "org.apache.lucene.analysis.";

    /** The log object for this class. */
    protected static final Log LOG = CmsLog.getLog(CmsSearchManager.class);

    /** The administrator OpenCms user context to access OpenCms VFS resources. */
    protected CmsObject m_adminCms;

    /** The list of indexes that are configured for offline index mode. */
    protected List<A_CmsSearchIndex> m_offlineIndexes;

    /** The thread used of offline indexing. */
    protected CmsSearchOfflineIndexThread m_offlineIndexThread;

    /** Configured analyzers for languages using &lt;analyzer&gt;. */
    private HashMap<Locale, CmsSearchAnalyzer> m_analyzers;

    /** A map of document factory configurations. */
    private List<CmsSearchDocumentType> m_documentTypeConfigs;

    /** A map of document factories keyed by their matching Cms resource types and/or mimetypes. */
    private Map<String, I_CmsDocumentFactory> m_documentTypes;

    /** The max age for extraction results to remain in the cache. */
    private float m_extractionCacheMaxAge;

    /** The cache for the extraction results. */
    private CmsExtractionResultCache m_extractionResultCache;

    /** Contains the available field configurations. */
    private Map<String, I_CmsSearchFieldConfiguration> m_fieldConfigurations;

    /** The force unlock type. */
    private CmsSearchForceUnlockMode m_forceUnlockMode;

    /** The class used to highlight the search terms in the excerpt of a search result. */
    private I_CmsTermHighlighter m_highlighter;

    /** A list of search indexes. */
    private List<A_CmsSearchIndex> m_indexes;

    /** Seconds to wait for an index lock. */
    private int m_indexLockMaxWaitSeconds = 10;

    /** Configured index sources. */
    private Map<String, CmsSearchIndexSource> m_indexSources;

    /** The max. char. length of the excerpt in the search result. */
    private int m_maxExcerptLength;

    /** The maximum number of modifications before a commit in the search index is triggered. */
    private int m_maxModificationsBeforeCommit;

    /** The offline index search handler. */
    private CmsSearchOfflineHandler m_offlineHandler;

    /** The update frequency of the offline indexer in milliseconds. */
    private long m_offlineUpdateFrequency;

    /** Path to index files below WEB-INF/. */
    private String m_path;

    /** The Solr configuration. */
    private CmsSolrConfiguration m_solrConfig;

    /** Timeout for abandoning indexing thread. */
    private long m_timeout;

    /**
     * Default constructor when called as cron job.<p>
     */
    public CmsSearchManager() {

        m_documentTypes = new HashMap<String, I_CmsDocumentFactory>();
        m_documentTypeConfigs = new ArrayList<CmsSearchDocumentType>();
        m_analyzers = new HashMap<Locale, CmsSearchAnalyzer>();
        m_indexes = new ArrayList<A_CmsSearchIndex>();
        m_indexSources = new TreeMap<String, CmsSearchIndexSource>();
        m_offlineHandler = new CmsSearchOfflineHandler();
        m_extractionCacheMaxAge = DEFAULT_EXTRACTION_CACHE_MAX_AGE;
        m_maxExcerptLength = DEFAULT_EXCERPT_LENGTH;
        m_offlineUpdateFrequency = DEFAULT_OFFLINE_UPDATE_FREQNENCY;
        m_maxModificationsBeforeCommit = DEFAULT_MAX_MODIFICATIONS_BEFORE_COMMIT;

        m_fieldConfigurations = new HashMap<String, I_CmsSearchFieldConfiguration>();
        // make sure we have a "standard" field configuration
        addFieldConfiguration(CmsSearchFieldConfiguration.DEFAULT_STANDARD);

        if (CmsLog.INIT.isInfoEnabled()) {
            CmsLog.INIT.info(Messages.get().getBundle().key(Messages.INIT_START_SEARCH_CONFIG_0));
        }
    }

    /**
     * Returns an analyzer for the given class name.<p>
     * 
     * @param className the class name of the analyzer
     * 
     * @return the appropriate lucene analyzer
     * 
     * @throws Exception if something goes wrong
     */
    public static Analyzer getAnalyzer(String className) throws Exception {

        return getAnalyzer(className, null);
    }

    /**
     * Returns an analyzer for the given class name.<p>
     * 
     * Since Lucene 3.0, many analyzers require a "version" parameter in the constructor and 
     * can not be created by a simple <code>newInstance()</code> call.
     * This method will create analyzers by name for the {@link CmsLuceneIndex#LUCENE_VERSION} version.<p>
     * 
     * @param className the class name of the analyzer
     * @param stemmer the optional stemmer parameter required for the snowball analyzer
     * 
     * @return the appropriate lucene analyzer
     * 
     * @throws Exception if something goes wrong
     * 
     * @deprecated The stemmer parameter is used only by the snownall analyzer, which is deprecated in Lucene 3.
     */
    @Deprecated
    public static Analyzer getAnalyzer(String className, String stemmer) throws Exception {

        Analyzer analyzer = null;
        Class<?> analyzerClass;
        try {
            analyzerClass = Class.forName(className);
        } catch (ClassNotFoundException e) {
            // allow Lucene standard classes to be written in a short form
            analyzerClass = Class.forName(LUCENE_ANALYZER + className);
        }

        // since Lucene 3.0 most analyzers need a "version" parameter and don't support an empty constructor
        if (StandardAnalyzer.class.equals(analyzerClass)) {
            // the Lucene standard analyzer is used
            analyzer = new StandardAnalyzer(CmsLuceneIndex.LUCENE_VERSION);
        } else if (CmsGallerySearchAnalyzer.class.equals(analyzerClass)) {
            // OpenCms gallery multiple language analyzer
            analyzer = new CmsGallerySearchAnalyzer(CmsLuceneIndex.LUCENE_VERSION);
        } else {
            boolean hasEmpty = false;
            boolean hasVersion = false;
            boolean hasVersionWithString = false;
            // another analyzer is used, check if we find a suitable constructor 
            Constructor<?>[] constructors = analyzerClass.getConstructors();
            for (int i = 0; i < constructors.length; i++) {
                Constructor<?> c = constructors[i];
                Class<?>[] parameters = c.getParameterTypes();
                if (parameters.length == 0) {
                    // an empty constructor has been found
                    hasEmpty = true;
                }
                if ((parameters.length == 1) && parameters[0].equals(Version.class)) {
                    // a constructor with a Lucene version parameter has been found
                    hasVersion = true;
                }
                if ((stemmer != null)
                    && (parameters.length == 2)
                    && parameters[0].equals(Version.class)
                    && parameters[1].equals(String.class)) {
                    // a constructor with a Lucene version parameter and a String has been found
                    hasVersionWithString = true;
                }
            }
            if (hasVersionWithString) {
                // a constructor with a Lucene version parameter and a String has been found
                analyzer = (Analyzer)analyzerClass.getDeclaredConstructor(new Class[] {Version.class, String.class}).newInstance(
                    CmsLuceneIndex.LUCENE_VERSION,
                    stemmer);
            } else if (hasVersion) {
                // a constructor with a Lucene version parameter has been found
                analyzer = (Analyzer)analyzerClass.getDeclaredConstructor(new Class[] {Version.class}).newInstance(
                    CmsLuceneIndex.LUCENE_VERSION);
            } else if (hasEmpty) {
                // an empty constructor has been found
                analyzer = (Analyzer)analyzerClass.newInstance();
            }
        }
        return analyzer;
    }

    /**
     * Adds an analyzer.<p>
     * 
     * @param analyzer an analyzer
     */
    public void addAnalyzer(CmsSearchAnalyzer analyzer) {

        m_analyzers.put(analyzer.getLocale(), analyzer);

        if (CmsLog.INIT.isInfoEnabled()) {
            CmsLog.INIT.info(Messages.get().getBundle().key(
                Messages.INIT_ADD_ANALYZER_2,
                analyzer.getLocale(),
                analyzer.getClassName()));
        }
    }

    /**
     * Adds a document type.<p>
     * 
     * @param documentType a document type<commit_msg>Added static method to determine whether a search index is a Lucene index.<commit_after>/*
 * This library is part of OpenCms -
 * the Open Source Content Management System
 *
 * Copyright (c) Alkacon Software GmbH (http://www.alkacon.com)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * For further information about Alkacon Software GmbH, please see the
 * company website: http://www.alkacon.com
 *
 * For further information about OpenCms, please see the
 * project website: http://www.opencms.org
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

package org.opencms.search;

import org.opencms.db.CmsDriverManager;
import org.opencms.db.CmsPublishedResource;
import org.opencms.file.CmsObject;
import org.opencms.file.CmsResource;
import org.opencms.file.CmsResourceFilter;
import org.opencms.i18n.CmsMessageContainer;
import org.opencms.loader.CmsLoaderException;
import org.opencms.main.CmsEvent;
import org.opencms.main.CmsException;
import org.opencms.main.CmsIllegalArgumentException;
import org.opencms.main.CmsIllegalStateException;
import org.opencms.main.CmsLog;
import org.opencms.main.I_CmsEventListener;
import org.opencms.main.OpenCms;
import org.opencms.report.CmsLogReport;
import org.opencms.report.I_CmsReport;
import org.opencms.scheduler.I_CmsScheduledJob;
import org.opencms.search.documents.A_CmsVfsDocument;
import org.opencms.search.documents.CmsExtractionResultCache;
import org.opencms.search.documents.I_CmsDocumentFactory;
import org.opencms.search.documents.I_CmsTermHighlighter;
import org.opencms.search.fields.A_CmsSearchFieldConfiguration;
import org.opencms.search.fields.CmsSearchField;
import org.opencms.search.fields.CmsSearchFieldConfiguration;
import org.opencms.search.fields.CmsSearchFieldMapping;
import org.opencms.search.fields.I_CmsSearchField;
import org.opencms.search.fields.I_CmsSearchFieldConfiguration;
import org.opencms.search.galleries.CmsGallerySearchAnalyzer;
import org.opencms.search.solr.CmsSolrConfiguration;
import org.opencms.search.solr.CmsSolrFieldConfiguration;
import org.opencms.search.solr.CmsSolrIndex;
import org.opencms.security.CmsRole;
import org.opencms.security.CmsRoleViolationException;
import org.opencms.util.A_CmsModeStringEnumeration;
import org.opencms.util.CmsStringUtil;
import org.opencms.util.CmsUUID;

import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;

import org.apache.commons.logging.Log;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.Version;

/**
 * Implements the general management and configuration of the search and 
 * indexing facilities in OpenCms.<p>
 * 
 * @since 6.0.0 
 */
public class CmsSearchManager implements I_CmsScheduledJob, I_CmsEventListener {

    /**
     *  Enumeration class for force unlock types.<p>
     */
    public static final class CmsSearchForceUnlockMode extends A_CmsModeStringEnumeration {

        /** Force unlock type "always". */
        public static final CmsSearchForceUnlockMode ALWAYS = new CmsSearchForceUnlockMode("always");

        /** Force unlock type "never". */
        public static final CmsSearchForceUnlockMode NEVER = new CmsSearchForceUnlockMode("never");

        /** Force unlock type "only full". */
        public static final CmsSearchForceUnlockMode ONLYFULL = new CmsSearchForceUnlockMode("onlyfull");

        /** Serializable version id. */
        private static final long serialVersionUID = 74746076708908673L;

        /**
         * Creates a new force unlock type with the given name.<p>
         * 
         * @param mode the mode id to use
         */
        protected CmsSearchForceUnlockMode(String mode) {

            super(mode);
        }

        /**
         * Returns the lock type for the given type value.<p>
         *  
         * @param type the type value to get the lock type for
         * 
         * @return the lock type for the given type value
         */
        public static CmsSearchForceUnlockMode valueOf(String type) {

            if (type.equals(ALWAYS.toString())) {
                return ALWAYS;
            } else if (type.equals(NEVER.toString())) {
                return NEVER;
            } else {
                return ONLYFULL;
            }
        }
    }

    /**
     * Handles offline index generation.<p>
     */
    protected class CmsSearchOfflineHandler implements I_CmsEventListener {

        /** Indicates if the event handlers for the offline search have been already registered. */
        private boolean m_isEventRegistered;

        /** The list of resources to index. */
        private List<CmsPublishedResource> m_resourcesToIndex;

        /**
         * Initializes the offline index handler.<p>
         */
        protected CmsSearchOfflineHandler() {

            m_resourcesToIndex = new ArrayList<CmsPublishedResource>();
        }

        /**
         * Implements the event listener of this class.<p>
         * 
         * @see org.opencms.main.I_CmsEventListener#cmsEvent(org.opencms.main.CmsEvent)
         */
        @SuppressWarnings("unchecked")
        public void cmsEvent(CmsEvent event) {

            switch (event.getType()) {
                case I_CmsEventListener.EVENT_PROPERTY_MODIFIED:
                case I_CmsEventListener.EVENT_RESOURCE_CREATED:
                case I_CmsEventListener.EVENT_RESOURCE_AND_PROPERTIES_MODIFIED:
                case I_CmsEventListener.EVENT_RESOURCE_MODIFIED:
                    Object change = event.getData().get(I_CmsEventListener.KEY_CHANGE);
                    if ((change != null) && change.equals(new Integer(CmsDriverManager.NOTHING_CHANGED))) {
                        // skip lock & unlock
                        return;
                    }
                    // a resource has been modified - offline indexes require (re)indexing
                    List<CmsResource> resources = Collections.singletonList((CmsResource)event.getData().get(
                        I_CmsEventListener.KEY_RESOURCE));
                    reIndexResources(resources);
                    break;
                case I_CmsEventListener.EVENT_RESOURCES_AND_PROPERTIES_MODIFIED:
                case I_CmsEventListener.EVENT_RESOURCE_MOVED:
                case I_CmsEventListener.EVENT_RESOURCE_DELETED:
                case I_CmsEventListener.EVENT_RESOURCE_COPIED:
                case I_CmsEventListener.EVENT_RESOURCES_MODIFIED:
                    // a list of resources has been modified - offline indexes require (re)indexing
                    reIndexResources((List<CmsResource>)event.getData().get(I_CmsEventListener.KEY_RESOURCES));
                    break;
                default:
                    // no operation
            }
        }

        /**
         * Adds a list of {@link CmsPublishedResource} objects to be indexed.<p>
         * 
         * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to be indexed
         */
        protected synchronized void addResourcesToIndex(List<CmsPublishedResource> resourcesToIndex) {

            m_resourcesToIndex.addAll(resourcesToIndex);
        }

        /**
         * Returns the list of {@link CmsPublishedResource} objects to index.<p>
         *
         * @return the resources to index
         */
        protected synchronized List<CmsPublishedResource> getResourcesToIndex() {

            List<CmsPublishedResource> result = m_resourcesToIndex;
            m_resourcesToIndex = new ArrayList<CmsPublishedResource>();
            return result;
        }

        /**
         * Initializes this offline search handler, registering the event handlers if required.<p>
         */
        protected void initialize() {

            if (m_offlineIndexes.size() > 0) {
                // there is at least one offline index configured
                if ((m_offlineIndexThread == null) || !m_offlineIndexThread.isAlive()) {
                    // create the offline indexing thread
                    m_offlineIndexThread = new CmsSearchOfflineIndexThread(this);
                    // start the offline index thread
                    m_offlineIndexThread.start();
                }
            } else {
                if ((m_offlineIndexThread != null) && m_offlineIndexThread.isAlive()) {
                    // no offline indexes but thread still running, stop the thread
                    m_offlineIndexThread.shutDown();
                    m_offlineIndexThread = null;
                }
            }
            // do this only in case there are offline indexes configured
            if (!m_isEventRegistered && (m_offlineIndexes.size() > 0)) {
                m_isEventRegistered = true;
                // register this object as event listener
                OpenCms.addCmsEventListener(this, new int[] {
                    I_CmsEventListener.EVENT_PROPERTY_MODIFIED,
                    I_CmsEventListener.EVENT_RESOURCE_CREATED,
                    I_CmsEventListener.EVENT_RESOURCE_AND_PROPERTIES_MODIFIED,
                    I_CmsEventListener.EVENT_RESOURCE_MODIFIED,
                    I_CmsEventListener.EVENT_RESOURCES_AND_PROPERTIES_MODIFIED,
                    I_CmsEventListener.EVENT_RESOURCE_MOVED,
                    I_CmsEventListener.EVENT_RESOURCE_DELETED,
                    I_CmsEventListener.EVENT_RESOURCE_COPIED,
                    I_CmsEventListener.EVENT_RESOURCES_MODIFIED});
            }
        }

        /**
         * Updates all offline indexes for the given list of {@link CmsResource} objects.<p>
         * 
         * @param resources a list of {@link CmsResource} objects to update in the offline indexes
         */
        protected synchronized void reIndexResources(List<CmsResource> resources) {

            List<CmsPublishedResource> resourcesToIndex = new ArrayList<CmsPublishedResource>(resources.size());
            for (CmsResource res : resources) {
                CmsPublishedResource pubRes = new CmsPublishedResource(res);
                resourcesToIndex.add(pubRes);
            }
            if (resourcesToIndex.size() > 0) {
                // add the resources found to the offline index thread
                addResourcesToIndex(resourcesToIndex);
            }
        }
    }

    /**
     * The offline indexer thread runs periodically and indexes all resources added by the event handler.<p>
     */
    protected class CmsSearchOfflineIndexThread extends Thread {

        /** The event handler that triggers this thread. */
        CmsSearchOfflineHandler m_handler;

        /** Indicates if this thread is still alive. */
        boolean m_isAlive;

        /** If true a manual update (after file upload) was triggered. */
        private boolean m_updateTriggered;

        /**
         * Constructor.<p>
         * 
         * @param handler the offline index event handler
         */
        protected CmsSearchOfflineIndexThread(CmsSearchOfflineHandler handler) {

            super("OpenCms: Offline Search Indexer");
            m_handler = handler;
        }

        /**
         * @see java.lang.Thread#interrupt()
         */
        @Override
        public void interrupt() {

            super.interrupt();
            m_updateTriggered = true;
        }

        /**
         * @see java.lang.Thread#run()
         */
        @Override
        public void run() {

            // create a log report for the output
            I_CmsReport report = new CmsLogReport(m_adminCms.getRequestContext().getLocale(), CmsSearchManager.class);
            long offlineUpdateFrequency = getOfflineUpdateFrequency();
            m_updateTriggered = false;
            try {
                while (m_isAlive) {
                    if (!m_updateTriggered) {
                        try {
                            sleep(offlineUpdateFrequency);
                        } catch (InterruptedException e) {
                            // continue the thread after interruption
                            if (!m_isAlive) {
                                // the thread has been shut down while sleeping
                                continue;
                            }
                            if (offlineUpdateFrequency != getOfflineUpdateFrequency()) {
                                // offline update frequency change - clear interrupt status
                                offlineUpdateFrequency = getOfflineUpdateFrequency();
                            }
                        }
                    }
                    if (m_isAlive) {
                        // set update trigger to false since we do the update now
                        m_updateTriggered = false;
                        // get list of resource to update
                        List<CmsPublishedResource> resourcesToIndex = getResourcesToIndex();
                        if (resourcesToIndex.size() > 0) {
                            // only start indexing if there is at least one resource
                            startOfflineUpdateThread(report, resourcesToIndex);
                        }
                        // this is just called to clear the interrupt status of the thread
                        interrupted();
                    }
                }
            } finally {
                // make sure that live status is reset in case of Exceptions
                m_isAlive = false;
            }

        }

        /**
         * @see java.lang.Thread#start()
         */
        @Override
        public synchronized void start() {

            m_isAlive = true;
            super.start();
        }

        /**
         * Obtains the list of resource to update in the offline index, 
         * then optimizes the list by removing duplicate entries.<p>
         * 
         * @return the list of resource to update in the offline index
         */
        protected List<CmsPublishedResource> getResourcesToIndex() {

            List<CmsPublishedResource> resourcesToIndex = m_handler.getResourcesToIndex();
            List<CmsPublishedResource> result = new ArrayList<CmsPublishedResource>(resourcesToIndex.size());

            for (CmsPublishedResource pubRes : resourcesToIndex) {
                int pos = result.indexOf(pubRes);
                if (pos < 0) {
                    // resource not already contained in the update list
                    result.add(pubRes);
                } else {
                    CmsPublishedResource curRes = result.get(pos);
                    if ((pubRes.getState() != curRes.getState())
                        || (pubRes.getMovedState() != curRes.getMovedState())
                        || !pubRes.getRootPath().equals(curRes.getRootPath())) {
                        // resource already in the update list but new state is different, so also add this
                        result.add(pubRes);
                    }
                }
            }

            return result;
        }

        /**
         * Shuts down this offline index thread.<p>
         */
        protected void shutDown() {

            m_isAlive = false;
            interrupt();
        }

        /**
         * Updates the offline search indexes for the given list of resources.<p>
         * 
         * @param report the report to write the index information to
         * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to index
         */
        protected void startOfflineUpdateThread(I_CmsReport report, List<CmsPublishedResource> resourcesToIndex) {

            CmsSearchOfflineIndexWorkThread thread = new CmsSearchOfflineIndexWorkThread(report, resourcesToIndex);
            long startTime = System.currentTimeMillis();
            long waitTime = getOfflineUpdateFrequency() / 2;
            if (LOG.isDebugEnabled()) {
                LOG.debug(Messages.get().getBundle().key(
                    Messages.LOG_OI_UPDATE_START_1,
                    Integer.valueOf(resourcesToIndex.size())));
            }
            thread.start();

            do {
                try {
                    // wait half the time of the offline index frequency for the thread to finish
                    thread.join(waitTime);
                } catch (InterruptedException e) {
                    // continue
                }
                if (thread.isAlive()) {
                    LOG.warn(Messages.get().getBundle().key(
                        Messages.LOG_OI_UPDATE_LONG_2,
                        Integer.valueOf(resourcesToIndex.size()),
                        Long.valueOf(System.currentTimeMillis() - startTime)));
                }
            } while (thread.isAlive());

            if (LOG.isDebugEnabled()) {
                LOG.debug(Messages.get().getBundle().key(
                    Messages.LOG_OI_UPDATE_FINISH_2,
                    Integer.valueOf(resourcesToIndex.size()),
                    Long.valueOf(System.currentTimeMillis() - startTime)));
            }
        }
    }

    /**
     * An offline index worker Thread runs each time for every offline index update action.<p>
     * 
     * This was decoupled from the main {@link CmsSearchOfflineIndexThread} in order to avoid 
     * problems if a single operation "hangs" the Tread.<p>
     */
    protected class CmsSearchOfflineIndexWorkThread extends Thread {

        /** The report to write the index information to. */
        I_CmsReport m_report;

        /** The list of {@link CmsPublishedResource} objects to index. */
        List<CmsPublishedResource> m_resourcesToIndex;

        /**
         * Updates the offline search indexes for the given list of resources.<p>
         * 
         * @param report the report to write the index information to
         * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to index
         */
        protected CmsSearchOfflineIndexWorkThread(I_CmsReport report, List<CmsPublishedResource> resourcesToIndex) {

            super("OpenCms: Offline Search Index Worker");
            m_report = report;
            m_resourcesToIndex = resourcesToIndex;
        }

        /**
         * @see java.lang.Thread#run()
         */
        @Override
        public void run() {

            updateIndexOffline(m_report, m_resourcesToIndex);
        }
    }

    /** The default value used for generating search result excerpts (1024 chars). */
    public static final int DEFAULT_EXCERPT_LENGTH = 1024;

    /** The default value used for keeping the extraction results in the cache (672 hours = 4 weeks). */
    public static final float DEFAULT_EXTRACTION_CACHE_MAX_AGE = 672.0f;

    /** Default for the maximum number of modifications before a commit in the search index is triggered (500). */
    public static final int DEFAULT_MAX_MODIFICATIONS_BEFORE_COMMIT = 500;

    /** The default update frequency for offline indexes (15000 msec = 15 sec). */
    public static final int DEFAULT_OFFLINE_UPDATE_FREQNENCY = 15000;

    /** The default timeout value used for generating a document for the search index (60000 msec = 1 min). */
    public static final int DEFAULT_TIMEOUT = 60000;

    /** Scheduler parameter: Update only a specified list of indexes. */
    public static final String JOB_PARAM_INDEXLIST = "indexList";

    /** Scheduler parameter: Write the output of the update to the logfile. */
    public static final String JOB_PARAM_WRITELOG = "writeLog";

    /** Prefix for Lucene default analyzers package (<code>org.apache.lucene.analysis.</code>). */
    public static final String LUCENE_ANALYZER = "org.apache.lucene.analysis.";

    /** The log object for this class. */
    protected static final Log LOG = CmsLog.getLog(CmsSearchManager.class);

    /** The administrator OpenCms user context to access OpenCms VFS resources. */
    protected CmsObject m_adminCms;

    /** The list of indexes that are configured for offline index mode. */
    protected List<A_CmsSearchIndex> m_offlineIndexes;

    /** The thread used of offline indexing. */
    protected CmsSearchOfflineIndexThread m_offlineIndexThread;

    /** Configured analyzers for languages using &lt;analyzer&gt;. */
    private HashMap<Locale, CmsSearchAnalyzer> m_analyzers;

    /** A map of document factory configurations. */
    private List<CmsSearchDocumentType> m_documentTypeConfigs;

    /** A map of document factories keyed by their matching Cms resource types and/or mimetypes. */
    private Map<String, I_CmsDocumentFactory> m_documentTypes;

    /** The max age for extraction results to remain in the cache. */
    private float m_extractionCacheMaxAge;

    /** The cache for the extraction results. */
    private CmsExtractionResultCache m_extractionResultCache;

    /** Contains the available field configurations. */
    private Map<String, I_CmsSearchFieldConfiguration> m_fieldConfigurations;

    /** The force unlock type. */
    private CmsSearchForceUnlockMode m_forceUnlockMode;

    /** The class used to highlight the search terms in the excerpt of a search result. */
    private I_CmsTermHighlighter m_highlighter;

    /** A list of search indexes. */
    private List<A_CmsSearchIndex> m_indexes;

    /** Seconds to wait for an index lock. */
    private int m_indexLockMaxWaitSeconds = 10;

    /** Configured index sources. */
    private Map<String, CmsSearchIndexSource> m_indexSources;

    /** The max. char. length of the excerpt in the search result. */
    private int m_maxExcerptLength;

    /** The maximum number of modifications before a commit in the search index is triggered. */
    private int m_maxModificationsBeforeCommit;

    /** The offline index search handler. */
    private CmsSearchOfflineHandler m_offlineHandler;

    /** The update frequency of the offline indexer in milliseconds. */
    private long m_offlineUpdateFrequency;

    /** Path to index files below WEB-INF/. */
    private String m_path;

    /** The Solr configuration. */
    private CmsSolrConfiguration m_solrConfig;

    /** Timeout for abandoning indexing thread. */
    private long m_timeout;

    /**
     * Default constructor when called as cron job.<p>
     */
    public CmsSearchManager() {

        m_documentTypes = new HashMap<String, I_CmsDocumentFactory>();
        m_documentTypeConfigs = new ArrayList<CmsSearchDocumentType>();
        m_analyzers = new HashMap<Locale, CmsSearchAnalyzer>();
        m_indexes = new ArrayList<A_CmsSearchIndex>();
        m_indexSources = new TreeMap<String, CmsSearchIndexSource>();
        m_offlineHandler = new CmsSearchOfflineHandler();
        m_extractionCacheMaxAge = DEFAULT_EXTRACTION_CACHE_MAX_AGE;
        m_maxExcerptLength = DEFAULT_EXCERPT_LENGTH;
        m_offlineUpdateFrequency = DEFAULT_OFFLINE_UPDATE_FREQNENCY;
        m_maxModificationsBeforeCommit = DEFAULT_MAX_MODIFICATIONS_BEFORE_COMMIT;

        m_fieldConfigurations = new HashMap<String, I_CmsSearchFieldConfiguration>();
        // make sure we have a "standard" field configuration
        addFieldConfiguration(CmsSearchFieldConfiguration.DEFAULT_STANDARD);

        if (CmsLog.INIT.isInfoEnabled()) {
            CmsLog.INIT.info(Messages.get().getBundle().key(Messages.INIT_START_SEARCH_CONFIG_0));
        }
    }

    /**
     * Returns an analyzer for the given class name.<p>
     * 
     * @param className the class name of the analyzer
     * 
     * @return the appropriate lucene analyzer
     * 
     * @throws Exception if something goes wrong
     */
    public static Analyzer getAnalyzer(String className) throws Exception {

        return getAnalyzer(className, null);
    }

    /**
     * Returns an analyzer for the given class name.<p>
     * 
     * Since Lucene 3.0, many analyzers require a "version" parameter in the constructor and 
     * can not be created by a simple <code>newInstance()</code> call.
     * This method will create analyzers by name for the {@link CmsLuceneIndex#LUCENE_VERSION} version.<p>
     * 
     * @param className the class name of the analyzer
     * @param stemmer the optional stemmer parameter required for the snowball analyzer
     * 
     * @return the appropriate lucene analyzer
     * 
     * @throws Exception if something goes wrong
     * 
     * @deprecated The stemmer parameter is used only by the snownall analyzer, which is deprecated in Lucene 3.
     */
    @Deprecated
    public static Analyzer getAnalyzer(String className, String stemmer) throws Exception {

        Analyzer analyzer = null;
        Class<?> analyzerClass;
        try {
            analyzerClass = Class.forName(className);
        } catch (ClassNotFoundException e) {
            // allow Lucene standard classes to be written in a short form
            analyzerClass = Class.forName(LUCENE_ANALYZER + className);
        }

        // since Lucene 3.0 most analyzers need a "version" parameter and don't support an empty constructor
        if (StandardAnalyzer.class.equals(analyzerClass)) {
            // the Lucene standard analyzer is used
            analyzer = new StandardAnalyzer(CmsLuceneIndex.LUCENE_VERSION);
        } else if (CmsGallerySearchAnalyzer.class.equals(analyzerClass)) {
            // OpenCms gallery multiple language analyzer
            analyzer = new CmsGallerySearchAnalyzer(CmsLuceneIndex.LUCENE_VERSION);
        } else {
            boolean hasEmpty = false;
            boolean hasVersion = false;
            boolean hasVersionWithString = false;
            // another analyzer is used, check if we find a suitable constructor 
            Constructor<?>[] constructors = analyzerClass.getConstructors();
            for (int i = 0; i < constructors.length; i++) {
                Constructor<?> c = constructors[i];
                Class<?>[] parameters = c.getParameterTypes();
                if (parameters.length == 0) {
                    // an empty constructor has been found
                    hasEmpty = true;
                }
                if ((parameters.length == 1) && parameters[0].equals(Version.class)) {
                    // a constructor with a Lucene version parameter has been found
                    hasVersion = true;
                }
                if ((stemmer != null)
                    && (parameters.length == 2)
                    && parameters[0].equals(Version.class)
                    && parameters[1].equals(String.class)) {
                    // a constructor with a Lucene version parameter and a String has been found
                    hasVersionWithString = true;
                }
            }
            if (hasVersionWithString) {
                // a constructor with a Lucene version parameter and a String has been found
                analyzer = (Analyzer)analyzerClass.getDeclaredConstructor(new Class[] {Version.class, String.class}).newInstance(
                    CmsLuceneIndex.LUCENE_VERSION,
                    stemmer);
            } else if (hasVersion) {
                // a constructor with a Lucene version parameter has been found
                analyzer = (Analyzer)analyzerClass.getDeclaredConstructor(new Class[] {Version.class}).newInstance(
                    CmsLuceneIndex.LUCENE_VERSION);
            } else if (hasEmpty) {
                // an empty constructor has been found
                analyzer = (Analyzer)analyzerClass.newInstance();
            }
        }
        return analyzer;
    }

    /**
     * Returns <code>true</code> if the index for the given name is a Lucene index, <code>false</code> otherwise.<p>
     * 
     * @param indexName the name of the index to check
     * 
     * @return <code>true</code> if the index for the given name is a Lucene index
     */
    public static boolean isLuceneIndex(String indexName) {

        return OpenCms.getSearchManager().getIndexLucene(indexName) != null ? true : false;
    }

    /**
     * Adds an analyzer.<p>
     * 
     * @param analyzer an analyzer
     */
    public void addAnalyzer(CmsSearchAnalyzer analyzer) {

        m_analyzers.put(analyzer.getLocale(), analyzer);

        if (CmsLog.INIT.isInfoEnabled()) {
            CmsLog.INIT.info(Messages.get().getBundle().key(
                Messages.INIT_ADD_ANALYZER_2,
                analyzer.getLocale(),
                analyzer.getClassName()));
        }
    }

    /**
     * Adds a document type.<p>
     * 
     * @param documentType a document type
{'fraction_non_alphanumeric': 0.07791827453142228, 'fraction_numerical': 0.003721058434399118, 'mean_word_length': 2.13321098936687, 'pattern_counts': {'":': 0, '<': 144, '<?xml version=': 0, '>': 148, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3944768', 'n_tokens_mistral': 15529, 'n_tokens_neox': 14498, 'n_words': 4176}