<commit_before>/*
 * Copyright 2004-2005 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.codehaus.groovy.grails.plugins;

import grails.util.Environment;
import grails.util.GrailsUtil;
import grails.util.Metadata;
import groovy.lang.Binding;
import groovy.lang.GroovyClassLoader;
import groovy.lang.GroovyRuntimeException;
import groovy.lang.GroovyShell;
import groovy.lang.GroovySystem;
import groovy.lang.MetaClassRegistry;
import groovy.lang.Writable;
import groovy.util.ConfigObject;
import groovy.util.ConfigSlurper;
import groovy.util.XmlSlurper;
import groovy.util.slurpersupport.GPathResult;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.io.Writer;
import java.lang.reflect.Modifier;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.servlet.ServletContext;
import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.codehaus.groovy.control.CompilationFailedException;
import org.codehaus.groovy.grails.commons.ConfigurationHolder;
import org.codehaus.groovy.grails.commons.GrailsApplication;
import org.codehaus.groovy.grails.commons.GrailsResourceUtils;
import org.codehaus.groovy.grails.commons.spring.WebRuntimeSpringConfiguration;
import org.codehaus.groovy.grails.plugins.exceptions.PluginException;
import org.codehaus.groovy.grails.support.ParentApplicationContextAware;
import org.codehaus.groovy.runtime.DefaultGroovyMethods;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.support.GenericApplicationContext;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.util.Assert;
import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

/**
 * <p>Handles the loading and management of plug-ins in the Grails system.
 * A plugin is just like a normal Grails application except that it contains a file ending
 * in *Plugin.groovy in the root of the directory.
 * <p/>
 * <p>A Plugin class is a Groovy class that has a version and optionally closures
 * called doWithSpring, doWithContext and doWithWebDescriptor
 * <p/>
 * <p>The doWithSpring closure uses the BeanBuilder syntax (@see grails.spring.BeanBuilder) to
 * provide runtime configuration of Grails via Spring
 * <p/>
 * <p>The doWithContext closure is called after the Spring ApplicationContext is built and accepts
 * a single argument (the ApplicationContext)
 * <p/>
 * <p>The doWithWebDescriptor uses mark-up building to provide additional functionality to the web.xml
 * file
 * <p/>
 * <p> Example:
 * <pre>
 * class ClassEditorGrailsPlugin {
 *      def version = '1.1'
 *      def doWithSpring = { application ->
 *          classEditor(org.springframework.beans.propertyeditors.ClassEditor, application.classLoader)
 *      }
 * }
 * </pre>
 * <p/>
 * <p>A plugin can also define "dependsOn" and "evict" properties that specify what plugins the plugin
 * depends on and which ones it is incompatable with and should evict
 *
 * @author <NAME>
 * @since 0.4
 */
public class DefaultGrailsPluginManager extends AbstractGrailsPluginManager implements GrailsPluginManager {

    private static final Log LOG = LogFactory.getLog(DefaultGrailsPluginManager.class);
    protected static final Class<?>[] COMMON_CLASSES = {
        Boolean.class, Byte.class, Character.class, Class.class, Double.class, Float.class,
        Integer.class, Long.class, Number.class, Short.class, String.class, BigInteger.class,
        BigDecimal.class, URL.class, URI.class };

    private GrailsPluginChangeChecker pluginChangeScanner;
    private static final int SCAN_INTERVAL = Integer.getInteger("grails.scan.interval", 5000).intValue(); //in ms

    private List<GrailsPlugin> delayedLoadPlugins = new LinkedList<GrailsPlugin>();
    private ApplicationContext parentCtx;
    private PathMatchingResourcePatternResolver resolver;
    private Map<GrailsPlugin, String[]> delayedEvictions = new HashMap<GrailsPlugin, String[]>();
    private ServletContext servletContext;
    private Map<String, Set<GrailsPlugin>> pluginToObserverMap = new HashMap<String, Set<GrailsPlugin>>();

    private long configLastModified;
    private PluginFilter pluginFilter;
    private static final String GRAILS_PLUGIN_SUFFIX = "GrailsPlugin";
    private List<GrailsPlugin> userPlugins = new ArrayList<GrailsPlugin>();

    public DefaultGrailsPluginManager(String resourcePath, GrailsApplication application) {
        super(application);
        Assert.notNull(application, "Argument [application] cannot be null!");

        resolver = new PathMatchingResourcePatternResolver();
        try {
            pluginResources = resolver.getResources(resourcePath);
        }
        catch (IOException ioe) {
            LOG.debug("Unable to load plugins for resource path " + resourcePath, ioe);
        }
        //corePlugins = new PathMatchingResourcePatternResolver().getResources("classpath:org/codehaus/groovy/grails/**/plugins/**GrailsPlugin.groovy");
        this.application = application;
        setPluginFilter();
    }

    public DefaultGrailsPluginManager(String[] pluginResources, GrailsApplication application) {
        super(application);
        resolver = new PathMatchingResourcePatternResolver();

        List<Resource> resourceList = new ArrayList<Resource>();
        for (String resourcePath : pluginResources) {
            try {
                for (Resource resource : resolver.getResources(resourcePath)) {
                    resourceList.add(resource);
                }
            }
            catch (IOException ioe) {
                LOG.debug("Unable to load plugins for resource path " + resourcePath, ioe);
            }
        }

        this.pluginResources = resourceList.toArray(new Resource[resourceList.size()]);
        this.application = application;
        setPluginFilter();
    }

    public DefaultGrailsPluginManager(Class<?>[] plugins, GrailsApplication application) {
        super(application);
        pluginClasses = plugins;
        resolver = new PathMatchingResourcePatternResolver();
        //this.corePlugins = new PathMatchingResourcePatternResolver().getResources("classpath:org/codehaus/groovy/grails/**/plugins/**GrailsPlugin.groovy");
        this.application = application;
        setPluginFilter();
    }

    public DefaultGrailsPluginManager(Resource[] pluginFiles, GrailsApplication application) {
        super(application);
        resolver = new PathMatchingResourcePatternResolver();
        pluginResources = pluginFiles;
        this.application = application;
        setPluginFilter();
    }

    public GrailsPlugin[] getUserPlugins() {
        return userPlugins.toArray(new GrailsPlugin[userPlugins.size()]);
    }

    private void setPluginFilter() {
        pluginFilter = new PluginFilterRetriever().getPluginFilter(application.getConfig());
    }

    public void startPluginChangeScanner() {
        if (pluginChangeScanner == null) {
            pluginChangeScanner = new GrailsPluginChangeChecker(this);
        }
        if (pluginChangeScanner.isAlive()) {
            throw new IllegalStateException("Plugin change scanner is already running!");
        }

        pluginChangeScanner.start();
        LOG.info("Started to scan for plugin changes in every " + SCAN_INTERVAL + "ms.");
    }

    public void stopPluginChangeScanner() {
        if (pluginChangeScanner != null) {
            pluginChangeScanner.enabled = false;
            try {
                try {
                    // wait for thread to die
                    pluginChangeScanner.join(5000);
                }
                catch (InterruptedException e) {
                    // ignore
                }
            }
            finally {
                pluginChangeScanner = new GrailsPluginChangeChecker(this);
            }
        }
    }

    public void refreshPlugin(String name) {
        if (hasGrailsPlugin(name)) {
            getGrailsPlugin(name).refresh();
        }
    }

    public Collection<GrailsPlugin> getPluginObservers(GrailsPlugin plugin) {
        Assert.notNull(plugin, "Argument [plugin] cannot be null");

        Collection<GrailsPlugin> c = pluginToObserverMap.get(plugin.getName());

        // Add any wildcard observers.
        Collection<GrailsPlugin> wildcardObservers = pluginToObserverMap.get("*");
        if (wildcardObservers != null) {
            if (c != null) {
                c.addAll(wildcardObservers);
            }
            else {
                c = wildcardObservers;
            }
        }

        if (c != null) {
            // Make sure this plugin is not observing itself!
            c.remove(plugin);
            return c;
        }

        return Collections.emptySet();
    }

    @SuppressWarnings("rawtypes")
    public void informObservers(String pluginName, Map event) {
        GrailsPlugin plugin = getGrailsPlugin(pluginName);
        if (plugin == null) {
            return;
        }

        for (GrailsPlugin observingPlugin : getPluginObservers(plugin)) {
            observingPlugin.notifyOfEvent(event);
        }
    }

    /* (non-Javadoc)
     * @see org.codehaus.groovy.grails.plugins.GrailsPluginManager#loadPlugins()
     */
    public void loadPlugins() throws PluginException {
        if (initialised) {
            return;
        }

        ClassLoader gcl = application.getClassLoader();

        attemptLoadPlugins(gcl);

        if (!delayedLoadPlugins.isEmpty()) {
            loadDelayedPlugins();
        }
        if (!delayedEvictions.isEmpty()) {
            processDelayedEvictions();
        }

        pluginList = sortPlugins(pluginList);
        initializePlugins();
        initialised = true;
    }

    private List<GrailsPlugin> sortPlugins(List<GrailsPlugin> toSort) {

        List<GrailsPlugin> newList = new ArrayList<GrailsPlugin>(toSort);

        for (GrailsPlugin plugin : toSort) {
            int i = newList.indexOf(plugin);
            for (String loadBefore : plugin.getLoadBeforeNames()) {
                final GrailsPlugin loadBeforePlugin = getGrailsPlugin(loadBefore);
                int j = newList.indexOf(loadBeforePlugin);
                if (i > j) {
                    newList.remove(plugin);
                    newList.add(j,plugin);
                }
            }

            i = newList.indexOf(plugin);
            for (String loadAfter : plugin.getLoadAfterNames()) {
                final GrailsPlugin loadAfterPlugin = getGrailsPlugin(loadAfter);
                int j = newList.indexOf(loadAfterPlugin);
                if (i < j) {
                    newList.remove(plugin);
                    newList.add(j,plugin);
                }
            }
        }
        return newList;
    }

    private void attemptLoadPlugins(ClassLoader gcl) {
        // retrieve load core plugins first
        List<GrailsPlugin>  grailsCorePlugins = loadCorePlugins ? findCorePlugins() : new ArrayList<GrailsPlugin>();
        List<GrailsPlugin>  grailsUserPlugins = findUserPlugins(gcl);
        userPlugins = grailsUserPlugins;

        List<GrailsPlugin>  allPlugins = new ArrayList<GrailsPlugin> (grailsCorePlugins);
        allPlugins.addAll(grailsUserPlugins);

        //filtering applies to user as well as core plugins
        List<GrailsPlugin> filteredPlugins = getPluginFilter().filterPluginList(allPlugins);

        //make sure core plugins are loaded first
        List<GrailsPlugin>  orderedCorePlugins = new ArrayList<GrailsPlugin> ();
        List<GrailsPlugin>  orderedUserPlugins = new ArrayList<GrailsPlugin> ();

        for (GrailsPlugin plugin : filteredPlugins) {
            if (grailsCorePlugins.contains(plugin)) {
                orderedCorePlugins.add(plugin);
            }
            else {
                orderedUserPlugins.add(plugin);
            }
        }

        List<GrailsPlugin>  orderedPlugins = new ArrayList<GrailsPlugin> ();
        orderedPlugins.addAll(orderedCorePlugins);
        orderedPlugins.addAll(orderedUserPlugins);

        for (GrailsPlugin plugin : orderedPlugins) {
            attemptPluginLoad(plugin);
        }
    }

    private List<GrailsPlugin> findCorePlugins() {
        CorePluginFinder finder = new CorePluginFinder(application);

        List<GrailsPlugin> grailsCorePlugins = new ArrayList<GrailsPlugin>();

        for (Class<?> pluginClass : finder.getPluginClasses()) {
            if (pluginClass != null && !Modifier.isAbstract(pluginClass.getModifiers()) && pluginClass != DefaultGrailsPlugin.class) {
                GrailsPlugin plugin = createGrailsPlugin(pluginClass);
                plugin.setApplicationContext(applicationContext);
                grailsCorePlugins.add(plugin);
            }
        }
        return grailsCorePlugins;
    }

	protected GrailsPlugin createGrailsPlugin(Class<?> pluginClass) {
		return new DefaultGrailsPlugin(pluginClass, application);
	}

	protected GrailsPlugin createGrailsPlugin(Class<?> pluginClass,
			Resource resource) {
		return new DefaultGrailsPlugin(pluginClass, resource, application);
	}
	
    private List<GrailsPlugin>  findUserPlugins(ClassLoader gcl) {
        List<GrailsPlugin>  grailsUserPlugins = new ArrayList<GrailsPlugin>();

        LOG.info("Attempting to load [" + pluginResources.length + "] user defined plugins");
        for (int i = 0; i < pluginResources.length; i++) {
            Resource r = pluginResources[i];

            Class<?> pluginClass = loadPluginClass(gcl, r);

            if (isGrailsPlugin(pluginClass)) {
                GrailsPlugin plugin = createGrailsPlugin(pluginClass, r);
                //attemptPluginLoad(plugin);
                grailsUserPlugins.add(plugin);
            }
            else {
                LOG.warn("Class [" + pluginClass + "] not loaded as plug-in. Grails plug-ins must end with the convention 'GrailsPlugin'!");
            }
        }

        for (int i = 0; i < pluginClasses.length; i++) {
            Class<?> pluginClass = pluginClasses[i];
            if (isGrailsPlugin(pluginClass)) {
                GrailsPlugin plugin = createGrailsPlugin(pluginClass);
                //attemptPluginLoad(plugin);
                grailsUserPlugins.add(plugin);
            }
            else {
                LOG.warn("Class [" + pluginClass + "] not loaded as plug-in. Grails plug-ins must end with the convention 'GrailsPlugin'!");
            }
        }
        return grailsUserPlugins;
    }


    private boolean isGrailsPlugin(Class<?> pluginClass) {
        return pluginClass != null && pluginClass.getName().endsWith(GRAILS_PLUGIN_SUFFIX);
    }

    private void processDelayedEvictions() {
        for (Map.Entry<GrailsPlugin, String[]> entry : delayedEvictions.entrySet()) {
            GrailsPlugin plugin = entry.getKey();
            for (String pluginName : entry.getValue()) {
                evictPlugin(plugin, pluginName);
            }
        }
    }

    private void initializePlugins() {
        for (Object plugin : plugins.values()) {
            if (plugin instanceof ApplicationContextAware) {
                ((ApplicationContextAware) plugin).setApplicationContext(applicationContext);
            }
        }
    }

    /**
     * This method will attempt to load that plug-ins not loaded in the first pass
     */
    private void loadDelayedPlugins() {
        while (!delayedLoadPlugins.isEmpty()) {
            GrailsPlugin plugin = delayedLoadPlugins.remove(0);
            if (areDependenciesResolved(plugin)) {
                if (!hasValidPluginsToLoadBefore(plugin)) {
                    registerPlugin(plugin);
                }
                else {
                    delayedLoadPlugins.add(plugin);
                }
            }
            else {
                // ok, it still hasn't resolved the dependency after the initial
                // load of all plugins. All hope is not lost, however, so lets first
                // look inside the remaining delayed loads before giving up
                boolean foundInDelayed = false;
                for (GrailsPlugin remainingPlugin : delayedLoadPlugins) {
                    if (isDependentOn(plugin, remainingPlugin)) {
                        foundInDelayed = true;
                        break;
                    }
                }
                if (foundInDelayed) {
                    delayedLoadPlugins.add(plugin);
                }
                else {
                    failedPlugins.put(plugin.getName(), plugin);
                    LOG.warn("WARNING: Plugin [" + plugin.getName() + "] cannot be loaded because its dependencies [" +
                            ArrayUtils.toString(plugin.getDependencyNames()) + "] cannot be resolved");
                }
            }
        }
    }

    private boolean hasValidPluginsToLoadBefore(GrailsPlugin plugin) {
        String[] loadAfterNames = plugin.getLoadAfterNames();
        for (Object delayedLoadPlugin : delayedLoadPlugins) {
            GrailsPlugin other = (GrailsPlugin) delayedLoadPlugin;
            for (String name : loadAfterNames) {
                if (other.getName().equals(name)) {
                    return hasDelayedDependencies(other) || areDependenciesResolved(other);
                }
            }
        }
        return false;
    }

    private boolean hasDelayedDependencies(GrailsPlugin other) {
        String[] dependencyNames = other.getDependencyNames();
        for (int i = 0; i < dependencyNames.length; i++) {
            String dependencyName = dependencyNames[i];
            for (GrailsPlugin grailsPlugin : delayedLoadPlugins) {
                if (grailsPlugin.getName().equals(dependencyName)) return true;
            }
        }
        return false;
    }

    /**
     * Checks whether the first plugin is dependant on the second plugin.
     *
     * @param plugin     The plugin to check
     * @param dependancy The plugin which the first argument may be dependant on
     * @return True if it is
     */
    private boolean isDependentOn(GrailsPlugin plugin, GrailsPlugin dependancy) {
        for (String name : plugin.getDependencyNames()) {
            String requiredVersion = plugin.getDependentVersion(name);

            if (name.equals(dependancy.getName()) &&
                    GrailsPluginUtils.isValidVersion(dependancy.getVersion(), requiredVersion))
                return true;
        }
        return false;
    }

    private boolean areDependenciesResolved(GrailsPlugin plugin) {
        for (String name : plugin.getDependencyNames()) {
            if (!hasGrailsPlugin(name, plugin.getDependentVersion(name))) {
                return false;
            }
        }
        return true;
    }

    /**
     * Returns true if there are no plugins left that should, if possible, be loaded before this plugin.
     *
     * @param plugin The plugin
     * @return True if there are
     */
    private boolean areNoneToLoadBefore(GrailsPlugin plugin) {
        for (String name : plugin.getLoadAfterNames()) {
            if (getGrailsPlugin(name) == null) {
                return false;
            }
        }
        return true;
    }

    private Class<?> loadPluginClass(ClassLoader cl, Resource r) {
        Class<?> pluginClass;
        if (cl instanceof GroovyClassLoader) {
            try {
                if (LOG.isInfoEnabled()) {
                    LOG.info("Parsing & compiling " + r.getFilename());
                }
                pluginClass = ((GroovyClassLoader)cl).parseClass(DefaultGroovyMethods.getText(r.getInputStream()));
            }
            catch (CompilationFailedException e) {
                throw new PluginException("Error compiling plugin [" + r.getFilename() + "] " + e.getMessage(), e);
            }
            catch (IOException e) {
                throw new PluginException("Error reading plugin [" + r.getFilename() + "] " + e.getMessage(), e);
            }
        }
        else {
            String className = GrailsResourceUtils.getClassName(r);
            try {
                pluginClass = Class.forName(className, true, cl);
            }
            catch (ClassNotFoundException e) {
                throw new PluginException("Cannot find plugin class [" + className + "] resource: [" + r.getFilename()+"]", e);
            }
        }
        return pluginClass;
    }

    /**
     * Attempts to load a plugin based on its dependencies. If a plugin's dependencies cannot be resolved
     * it will add it to the list of dependencies to be resolved later.
     *
     * @param plugin The plugin
     */
    private void attemptPluginLoad(GrailsPlugin plugin) {
        if (areDependenciesResolved(plugin) && areNoneToLoadBefore(plugin)) {
            registerPlugin(plugin);
        }
        else {
            delayedLoadPlugins.add(plugin);
        }
    }

    private void registerPlugin(GrailsPlugin plugin) {
        if (canRegisterPlugin(plugin)) {
            if (LOG.isInfoEnabled()) {
                LOG.info("Grails plug-in [" + plugin.getName() + "] with version [" + plugin.getVersion() + "] loaded successfully");
            }

            if (plugin instanceof ParentApplicationContextAware) {
                ((ParentApplicationContextAware) plugin).setParentApplicationContext(parentCtx);
            }
            plugin.setManager(this);
            String[] evictionNames = plugin.getEvictionNames();
            if (evictionNames.length > 0) {
                delayedEvictions.put(plugin, evictionNames);
            }

            String[] observedPlugins = plugin.getObservedPluginNames();
            for (int i = 0; i < observedPlugins.length; i++) {
                String observedPlugin = observedPlugins[i];
                Set<GrailsPlugin> observers = pluginToObserverMap.get(observedPlugin);
                if (observers == null) {
                    observers = new HashSet<GrailsPlugin>();
                    pluginToObserverMap.put(observedPlugin, observers);
                }
                observers.add(plugin);
            }
            pluginList.add(plugin);
            plugins.put(plugin.getName(), plugin);
            classNameToPluginMap.put(plugin.getPluginClass().getName(), plugin);
        }
        else {
            if (LOG.isInfoEnabled()) {
                LOG.info("Grails plugin " + plugin + " is disabled and was not loaded");
            }
        }
    }

    protected boolean canRegisterPlugin(GrailsPlugin plugin) {
        Environment environment = Environment.getCurrent();
        return plugin.isEnabled() && plugin.supportsEnvironment(environment);
    }

    protected void evictPlugin(GrailsPlugin evictor, String evicteeName) {
        GrailsPlugin pluginToEvict = plugins.get(evicteeName);
        if (pluginToEvict != null) {
            pluginList.remove(pluginToEvict);
            plugins.remove(pluginToEvict.getName());

            if (LOG.isInfoEnabled()) {
                LOG.info("Grails plug-in " + pluginToEvict + " was evicted by " + evictor);
            }
        }
    }

    private boolean hasGrailsPlugin(String name, String version) {
        return getGrailsPlugin(name, version) != null;
    }

    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
        this.applicationContext = applicationContext;
        for (GrailsPlugin plugin : pluginList) {
            plugin.setApplicationContext(applicationContext);
        }
    }

    public void setParentApplicationContext(ApplicationContext parent) {
        parentCtx = parent;
    }

    public void checkForChanges() {
        checkForConfigChanges();
        for (GrailsPlugin plugin : pluginList) {
            if (plugin.checkForChanges()) {
                LOG.info("Plugin " + plugin + " changed, re-registering beans...");
                reloadPlugin(plugin);
            }
        }
    }

    private void checkForConfigChanges() {
        ConfigObject config = application.getConfig();
        URL configURL = config.getConfigFile();
        if (configURL != null) {
            URLConnection connection;
            try {
                connection = configURL.openConnection();
            }
            catch (IOException e) {
                LOG.error("I/O error obtaining URL connection for configuration [" + configURL + "]: " + e.getMessage(), e);
                return;
            }

            try {
                long lastModified = connection.getLastModified();
                if (configLastModified == 0) {
                    configLastModified = lastModified;
                }
                else {
                    if (configLastModified < lastModified) {
                        LOG.info("Configuration [" + configURL + "] changed, reloading changes..");
                        ConfigSlurper slurper = new ConfigSlurper(Environment.getCurrent().getName());

                        try {
                            config = slurper.parse(configURL);
                            ConfigurationHolder.setConfig(config);
                            configLastModified = lastModified;
                            application.configChanged();
                            informPluginsOfConfigChange();
                        }
                        catch (GroovyRuntimeException gre) {
                            LOG.error("Unable to reload configuration. Please correct problem and try again: " + gre.getMessage(), gre);<commit_msg>GRAILS-7182 - When reloading the config during development, also reload any specified externalised config.<commit_after>/*
 * Copyright 2004-2005 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.codehaus.groovy.grails.plugins;

import grails.util.Environment;
import grails.util.GrailsUtil;
import grails.util.Metadata;
import groovy.lang.Binding;
import groovy.lang.GroovyClassLoader;
import groovy.lang.GroovyRuntimeException;
import groovy.lang.GroovyShell;
import groovy.lang.GroovySystem;
import groovy.lang.MetaClassRegistry;
import groovy.lang.Writable;
import groovy.util.ConfigObject;
import groovy.util.ConfigSlurper;
import groovy.util.XmlSlurper;
import groovy.util.slurpersupport.GPathResult;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.io.Writer;
import java.lang.reflect.Modifier;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.servlet.ServletContext;
import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.codehaus.groovy.control.CompilationFailedException;
import org.codehaus.groovy.grails.commons.ConfigurationHolder;
import org.codehaus.groovy.grails.commons.cfg.ConfigurationHelper;
import org.codehaus.groovy.grails.commons.GrailsApplication;
import org.codehaus.groovy.grails.commons.GrailsResourceUtils;
import org.codehaus.groovy.grails.commons.spring.WebRuntimeSpringConfiguration;
import org.codehaus.groovy.grails.plugins.exceptions.PluginException;
import org.codehaus.groovy.grails.support.ParentApplicationContextAware;
import org.codehaus.groovy.runtime.DefaultGroovyMethods;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.support.GenericApplicationContext;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.util.Assert;
import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

/**
 * <p>Handles the loading and management of plug-ins in the Grails system.
 * A plugin is just like a normal Grails application except that it contains a file ending
 * in *Plugin.groovy in the root of the directory.
 * <p/>
 * <p>A Plugin class is a Groovy class that has a version and optionally closures
 * called doWithSpring, doWithContext and doWithWebDescriptor
 * <p/>
 * <p>The doWithSpring closure uses the BeanBuilder syntax (@see grails.spring.BeanBuilder) to
 * provide runtime configuration of Grails via Spring
 * <p/>
 * <p>The doWithContext closure is called after the Spring ApplicationContext is built and accepts
 * a single argument (the ApplicationContext)
 * <p/>
 * <p>The doWithWebDescriptor uses mark-up building to provide additional functionality to the web.xml
 * file
 * <p/>
 * <p> Example:
 * <pre>
 * class ClassEditorGrailsPlugin {
 *      def version = '1.1'
 *      def doWithSpring = { application ->
 *          classEditor(org.springframework.beans.propertyeditors.ClassEditor, application.classLoader)
 *      }
 * }
 * </pre>
 * <p/>
 * <p>A plugin can also define "dependsOn" and "evict" properties that specify what plugins the plugin
 * depends on and which ones it is incompatable with and should evict
 *
 * @author <NAME>
 * @since 0.4
 */
public class DefaultGrailsPluginManager extends AbstractGrailsPluginManager implements GrailsPluginManager {

    private static final Log LOG = LogFactory.getLog(DefaultGrailsPluginManager.class);
    protected static final Class<?>[] COMMON_CLASSES = {
        Boolean.class, Byte.class, Character.class, Class.class, Double.class, Float.class,
        Integer.class, Long.class, Number.class, Short.class, String.class, BigInteger.class,
        BigDecimal.class, URL.class, URI.class };

    private GrailsPluginChangeChecker pluginChangeScanner;
    private static final int SCAN_INTERVAL = Integer.getInteger("grails.scan.interval", 5000).intValue(); //in ms

    private List<GrailsPlugin> delayedLoadPlugins = new LinkedList<GrailsPlugin>();
    private ApplicationContext parentCtx;
    private PathMatchingResourcePatternResolver resolver;
    private Map<GrailsPlugin, String[]> delayedEvictions = new HashMap<GrailsPlugin, String[]>();
    private ServletContext servletContext;
    private Map<String, Set<GrailsPlugin>> pluginToObserverMap = new HashMap<String, Set<GrailsPlugin>>();

    private long configLastModified;
    private PluginFilter pluginFilter;
    private static final String GRAILS_PLUGIN_SUFFIX = "GrailsPlugin";
    private List<GrailsPlugin> userPlugins = new ArrayList<GrailsPlugin>();

    public DefaultGrailsPluginManager(String resourcePath, GrailsApplication application) {
        super(application);
        Assert.notNull(application, "Argument [application] cannot be null!");

        resolver = new PathMatchingResourcePatternResolver();
        try {
            pluginResources = resolver.getResources(resourcePath);
        }
        catch (IOException ioe) {
            LOG.debug("Unable to load plugins for resource path " + resourcePath, ioe);
        }
        //corePlugins = new PathMatchingResourcePatternResolver().getResources("classpath:org/codehaus/groovy/grails/**/plugins/**GrailsPlugin.groovy");
        this.application = application;
        setPluginFilter();
    }

    public DefaultGrailsPluginManager(String[] pluginResources, GrailsApplication application) {
        super(application);
        resolver = new PathMatchingResourcePatternResolver();

        List<Resource> resourceList = new ArrayList<Resource>();
        for (String resourcePath : pluginResources) {
            try {
                for (Resource resource : resolver.getResources(resourcePath)) {
                    resourceList.add(resource);
                }
            }
            catch (IOException ioe) {
                LOG.debug("Unable to load plugins for resource path " + resourcePath, ioe);
            }
        }

        this.pluginResources = resourceList.toArray(new Resource[resourceList.size()]);
        this.application = application;
        setPluginFilter();
    }

    public DefaultGrailsPluginManager(Class<?>[] plugins, GrailsApplication application) {
        super(application);
        pluginClasses = plugins;
        resolver = new PathMatchingResourcePatternResolver();
        //this.corePlugins = new PathMatchingResourcePatternResolver().getResources("classpath:org/codehaus/groovy/grails/**/plugins/**GrailsPlugin.groovy");
        this.application = application;
        setPluginFilter();
    }

    public DefaultGrailsPluginManager(Resource[] pluginFiles, GrailsApplication application) {
        super(application);
        resolver = new PathMatchingResourcePatternResolver();
        pluginResources = pluginFiles;
        this.application = application;
        setPluginFilter();
    }

    public GrailsPlugin[] getUserPlugins() {
        return userPlugins.toArray(new GrailsPlugin[userPlugins.size()]);
    }

    private void setPluginFilter() {
        pluginFilter = new PluginFilterRetriever().getPluginFilter(application.getConfig());
    }

    public void startPluginChangeScanner() {
        if (pluginChangeScanner == null) {
            pluginChangeScanner = new GrailsPluginChangeChecker(this);
        }
        if (pluginChangeScanner.isAlive()) {
            throw new IllegalStateException("Plugin change scanner is already running!");
        }

        pluginChangeScanner.start();
        LOG.info("Started to scan for plugin changes in every " + SCAN_INTERVAL + "ms.");
    }

    public void stopPluginChangeScanner() {
        if (pluginChangeScanner != null) {
            pluginChangeScanner.enabled = false;
            try {
                try {
                    // wait for thread to die
                    pluginChangeScanner.join(5000);
                }
                catch (InterruptedException e) {
                    // ignore
                }
            }
            finally {
                pluginChangeScanner = new GrailsPluginChangeChecker(this);
            }
        }
    }

    public void refreshPlugin(String name) {
        if (hasGrailsPlugin(name)) {
            getGrailsPlugin(name).refresh();
        }
    }

    public Collection<GrailsPlugin> getPluginObservers(GrailsPlugin plugin) {
        Assert.notNull(plugin, "Argument [plugin] cannot be null");

        Collection<GrailsPlugin> c = pluginToObserverMap.get(plugin.getName());

        // Add any wildcard observers.
        Collection<GrailsPlugin> wildcardObservers = pluginToObserverMap.get("*");
        if (wildcardObservers != null) {
            if (c != null) {
                c.addAll(wildcardObservers);
            }
            else {
                c = wildcardObservers;
            }
        }

        if (c != null) {
            // Make sure this plugin is not observing itself!
            c.remove(plugin);
            return c;
        }

        return Collections.emptySet();
    }

    @SuppressWarnings("rawtypes")
    public void informObservers(String pluginName, Map event) {
        GrailsPlugin plugin = getGrailsPlugin(pluginName);
        if (plugin == null) {
            return;
        }

        for (GrailsPlugin observingPlugin : getPluginObservers(plugin)) {
            observingPlugin.notifyOfEvent(event);
        }
    }

    /* (non-Javadoc)
     * @see org.codehaus.groovy.grails.plugins.GrailsPluginManager#loadPlugins()
     */
    public void loadPlugins() throws PluginException {
        if (initialised) {
            return;
        }

        ClassLoader gcl = application.getClassLoader();

        attemptLoadPlugins(gcl);

        if (!delayedLoadPlugins.isEmpty()) {
            loadDelayedPlugins();
        }
        if (!delayedEvictions.isEmpty()) {
            processDelayedEvictions();
        }

        pluginList = sortPlugins(pluginList);
        initializePlugins();
        initialised = true;
    }

    private List<GrailsPlugin> sortPlugins(List<GrailsPlugin> toSort) {

        List<GrailsPlugin> newList = new ArrayList<GrailsPlugin>(toSort);

        for (GrailsPlugin plugin : toSort) {
            int i = newList.indexOf(plugin);
            for (String loadBefore : plugin.getLoadBeforeNames()) {
                final GrailsPlugin loadBeforePlugin = getGrailsPlugin(loadBefore);
                int j = newList.indexOf(loadBeforePlugin);
                if (i > j) {
                    newList.remove(plugin);
                    newList.add(j,plugin);
                }
            }

            i = newList.indexOf(plugin);
            for (String loadAfter : plugin.getLoadAfterNames()) {
                final GrailsPlugin loadAfterPlugin = getGrailsPlugin(loadAfter);
                int j = newList.indexOf(loadAfterPlugin);
                if (i < j) {
                    newList.remove(plugin);
                    newList.add(j,plugin);
                }
            }
        }
        return newList;
    }

    private void attemptLoadPlugins(ClassLoader gcl) {
        // retrieve load core plugins first
        List<GrailsPlugin>  grailsCorePlugins = loadCorePlugins ? findCorePlugins() : new ArrayList<GrailsPlugin>();
        List<GrailsPlugin>  grailsUserPlugins = findUserPlugins(gcl);
        userPlugins = grailsUserPlugins;

        List<GrailsPlugin>  allPlugins = new ArrayList<GrailsPlugin> (grailsCorePlugins);
        allPlugins.addAll(grailsUserPlugins);

        //filtering applies to user as well as core plugins
        List<GrailsPlugin> filteredPlugins = getPluginFilter().filterPluginList(allPlugins);

        //make sure core plugins are loaded first
        List<GrailsPlugin>  orderedCorePlugins = new ArrayList<GrailsPlugin> ();
        List<GrailsPlugin>  orderedUserPlugins = new ArrayList<GrailsPlugin> ();

        for (GrailsPlugin plugin : filteredPlugins) {
            if (grailsCorePlugins.contains(plugin)) {
                orderedCorePlugins.add(plugin);
            }
            else {
                orderedUserPlugins.add(plugin);
            }
        }

        List<GrailsPlugin>  orderedPlugins = new ArrayList<GrailsPlugin> ();
        orderedPlugins.addAll(orderedCorePlugins);
        orderedPlugins.addAll(orderedUserPlugins);

        for (GrailsPlugin plugin : orderedPlugins) {
            attemptPluginLoad(plugin);
        }
    }

    private List<GrailsPlugin> findCorePlugins() {
        CorePluginFinder finder = new CorePluginFinder(application);

        List<GrailsPlugin> grailsCorePlugins = new ArrayList<GrailsPlugin>();

        for (Class<?> pluginClass : finder.getPluginClasses()) {
            if (pluginClass != null && !Modifier.isAbstract(pluginClass.getModifiers()) && pluginClass != DefaultGrailsPlugin.class) {
                GrailsPlugin plugin = createGrailsPlugin(pluginClass);
                plugin.setApplicationContext(applicationContext);
                grailsCorePlugins.add(plugin);
            }
        }
        return grailsCorePlugins;
    }

	protected GrailsPlugin createGrailsPlugin(Class<?> pluginClass) {
		return new DefaultGrailsPlugin(pluginClass, application);
	}

	protected GrailsPlugin createGrailsPlugin(Class<?> pluginClass,
			Resource resource) {
		return new DefaultGrailsPlugin(pluginClass, resource, application);
	}
	
    private List<GrailsPlugin>  findUserPlugins(ClassLoader gcl) {
        List<GrailsPlugin>  grailsUserPlugins = new ArrayList<GrailsPlugin>();

        LOG.info("Attempting to load [" + pluginResources.length + "] user defined plugins");
        for (int i = 0; i < pluginResources.length; i++) {
            Resource r = pluginResources[i];

            Class<?> pluginClass = loadPluginClass(gcl, r);

            if (isGrailsPlugin(pluginClass)) {
                GrailsPlugin plugin = createGrailsPlugin(pluginClass, r);
                //attemptPluginLoad(plugin);
                grailsUserPlugins.add(plugin);
            }
            else {
                LOG.warn("Class [" + pluginClass + "] not loaded as plug-in. Grails plug-ins must end with the convention 'GrailsPlugin'!");
            }
        }

        for (int i = 0; i < pluginClasses.length; i++) {
            Class<?> pluginClass = pluginClasses[i];
            if (isGrailsPlugin(pluginClass)) {
                GrailsPlugin plugin = createGrailsPlugin(pluginClass);
                //attemptPluginLoad(plugin);
                grailsUserPlugins.add(plugin);
            }
            else {
                LOG.warn("Class [" + pluginClass + "] not loaded as plug-in. Grails plug-ins must end with the convention 'GrailsPlugin'!");
            }
        }
        return grailsUserPlugins;
    }


    private boolean isGrailsPlugin(Class<?> pluginClass) {
        return pluginClass != null && pluginClass.getName().endsWith(GRAILS_PLUGIN_SUFFIX);
    }

    private void processDelayedEvictions() {
        for (Map.Entry<GrailsPlugin, String[]> entry : delayedEvictions.entrySet()) {
            GrailsPlugin plugin = entry.getKey();
            for (String pluginName : entry.getValue()) {
                evictPlugin(plugin, pluginName);
            }
        }
    }

    private void initializePlugins() {
        for (Object plugin : plugins.values()) {
            if (plugin instanceof ApplicationContextAware) {
                ((ApplicationContextAware) plugin).setApplicationContext(applicationContext);
            }
        }
    }

    /**
     * This method will attempt to load that plug-ins not loaded in the first pass
     */
    private void loadDelayedPlugins() {
        while (!delayedLoadPlugins.isEmpty()) {
            GrailsPlugin plugin = delayedLoadPlugins.remove(0);
            if (areDependenciesResolved(plugin)) {
                if (!hasValidPluginsToLoadBefore(plugin)) {
                    registerPlugin(plugin);
                }
                else {
                    delayedLoadPlugins.add(plugin);
                }
            }
            else {
                // ok, it still hasn't resolved the dependency after the initial
                // load of all plugins. All hope is not lost, however, so lets first
                // look inside the remaining delayed loads before giving up
                boolean foundInDelayed = false;
                for (GrailsPlugin remainingPlugin : delayedLoadPlugins) {
                    if (isDependentOn(plugin, remainingPlugin)) {
                        foundInDelayed = true;
                        break;
                    }
                }
                if (foundInDelayed) {
                    delayedLoadPlugins.add(plugin);
                }
                else {
                    failedPlugins.put(plugin.getName(), plugin);
                    LOG.warn("WARNING: Plugin [" + plugin.getName() + "] cannot be loaded because its dependencies [" +
                            ArrayUtils.toString(plugin.getDependencyNames()) + "] cannot be resolved");
                }
            }
        }
    }

    private boolean hasValidPluginsToLoadBefore(GrailsPlugin plugin) {
        String[] loadAfterNames = plugin.getLoadAfterNames();
        for (Object delayedLoadPlugin : delayedLoadPlugins) {
            GrailsPlugin other = (GrailsPlugin) delayedLoadPlugin;
            for (String name : loadAfterNames) {
                if (other.getName().equals(name)) {
                    return hasDelayedDependencies(other) || areDependenciesResolved(other);
                }
            }
        }
        return false;
    }

    private boolean hasDelayedDependencies(GrailsPlugin other) {
        String[] dependencyNames = other.getDependencyNames();
        for (int i = 0; i < dependencyNames.length; i++) {
            String dependencyName = dependencyNames[i];
            for (GrailsPlugin grailsPlugin : delayedLoadPlugins) {
                if (grailsPlugin.getName().equals(dependencyName)) return true;
            }
        }
        return false;
    }

    /**
     * Checks whether the first plugin is dependant on the second plugin.
     *
     * @param plugin     The plugin to check
     * @param dependancy The plugin which the first argument may be dependant on
     * @return True if it is
     */
    private boolean isDependentOn(GrailsPlugin plugin, GrailsPlugin dependancy) {
        for (String name : plugin.getDependencyNames()) {
            String requiredVersion = plugin.getDependentVersion(name);

            if (name.equals(dependancy.getName()) &&
                    GrailsPluginUtils.isValidVersion(dependancy.getVersion(), requiredVersion))
                return true;
        }
        return false;
    }

    private boolean areDependenciesResolved(GrailsPlugin plugin) {
        for (String name : plugin.getDependencyNames()) {
            if (!hasGrailsPlugin(name, plugin.getDependentVersion(name))) {
                return false;
            }
        }
        return true;
    }

    /**
     * Returns true if there are no plugins left that should, if possible, be loaded before this plugin.
     *
     * @param plugin The plugin
     * @return True if there are
     */
    private boolean areNoneToLoadBefore(GrailsPlugin plugin) {
        for (String name : plugin.getLoadAfterNames()) {
            if (getGrailsPlugin(name) == null) {
                return false;
            }
        }
        return true;
    }

    private Class<?> loadPluginClass(ClassLoader cl, Resource r) {
        Class<?> pluginClass;
        if (cl instanceof GroovyClassLoader) {
            try {
                if (LOG.isInfoEnabled()) {
                    LOG.info("Parsing & compiling " + r.getFilename());
                }
                pluginClass = ((GroovyClassLoader)cl).parseClass(DefaultGroovyMethods.getText(r.getInputStream()));
            }
            catch (CompilationFailedException e) {
                throw new PluginException("Error compiling plugin [" + r.getFilename() + "] " + e.getMessage(), e);
            }
            catch (IOException e) {
                throw new PluginException("Error reading plugin [" + r.getFilename() + "] " + e.getMessage(), e);
            }
        }
        else {
            String className = GrailsResourceUtils.getClassName(r);
            try {
                pluginClass = Class.forName(className, true, cl);
            }
            catch (ClassNotFoundException e) {
                throw new PluginException("Cannot find plugin class [" + className + "] resource: [" + r.getFilename()+"]", e);
            }
        }
        return pluginClass;
    }

    /**
     * Attempts to load a plugin based on its dependencies. If a plugin's dependencies cannot be resolved
     * it will add it to the list of dependencies to be resolved later.
     *
     * @param plugin The plugin
     */
    private void attemptPluginLoad(GrailsPlugin plugin) {
        if (areDependenciesResolved(plugin) && areNoneToLoadBefore(plugin)) {
            registerPlugin(plugin);
        }
        else {
            delayedLoadPlugins.add(plugin);
        }
    }

    private void registerPlugin(GrailsPlugin plugin) {
        if (canRegisterPlugin(plugin)) {
            if (LOG.isInfoEnabled()) {
                LOG.info("Grails plug-in [" + plugin.getName() + "] with version [" + plugin.getVersion() + "] loaded successfully");
            }

            if (plugin instanceof ParentApplicationContextAware) {
                ((ParentApplicationContextAware) plugin).setParentApplicationContext(parentCtx);
            }
            plugin.setManager(this);
            String[] evictionNames = plugin.getEvictionNames();
            if (evictionNames.length > 0) {
                delayedEvictions.put(plugin, evictionNames);
            }

            String[] observedPlugins = plugin.getObservedPluginNames();
            for (int i = 0; i < observedPlugins.length; i++) {
                String observedPlugin = observedPlugins[i];
                Set<GrailsPlugin> observers = pluginToObserverMap.get(observedPlugin);
                if (observers == null) {
                    observers = new HashSet<GrailsPlugin>();
                    pluginToObserverMap.put(observedPlugin, observers);
                }
                observers.add(plugin);
            }
            pluginList.add(plugin);
            plugins.put(plugin.getName(), plugin);
            classNameToPluginMap.put(plugin.getPluginClass().getName(), plugin);
        }
        else {
            if (LOG.isInfoEnabled()) {
                LOG.info("Grails plugin " + plugin + " is disabled and was not loaded");
            }
        }
    }

    protected boolean canRegisterPlugin(GrailsPlugin plugin) {
        Environment environment = Environment.getCurrent();
        return plugin.isEnabled() && plugin.supportsEnvironment(environment);
    }

    protected void evictPlugin(GrailsPlugin evictor, String evicteeName) {
        GrailsPlugin pluginToEvict = plugins.get(evicteeName);
        if (pluginToEvict != null) {
            pluginList.remove(pluginToEvict);
            plugins.remove(pluginToEvict.getName());

            if (LOG.isInfoEnabled()) {
                LOG.info("Grails plug-in " + pluginToEvict + " was evicted by " + evictor);
            }
        }
    }

    private boolean hasGrailsPlugin(String name, String version) {
        return getGrailsPlugin(name, version) != null;
    }

    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
        this.applicationContext = applicationContext;
        for (GrailsPlugin plugin : pluginList) {
            plugin.setApplicationContext(applicationContext);
        }
    }

    public void setParentApplicationContext(ApplicationContext parent) {
        parentCtx = parent;
    }

    public void checkForChanges() {
        checkForConfigChanges();
        for (GrailsPlugin plugin : pluginList) {
            if (plugin.checkForChanges()) {
                LOG.info("Plugin " + plugin + " changed, re-registering beans...");
                reloadPlugin(plugin);
            }
        }
    }

    private void checkForConfigChanges() {
        ConfigObject config = application.getConfig();
        URL configURL = config.getConfigFile();
        if (configURL != null) {
            URLConnection connection;
            try {
                connection = configURL.openConnection();
            }
            catch (IOException e) {
                LOG.error("I/O error obtaining URL connection for configuration [" + configURL + "]: " + e.getMessage(), e);
                return;
            }

            try {
                long lastModified = connection.getLastModified();
                if (configLastModified == 0) {
                    configLastModified = lastModified;
                }
                else {
                    if (configLastModified < lastModified) {
                        LOG.info("Configuration [" + configURL + "] changed, reloading changes..");
                        ConfigSlurper slurper = new ConfigSlurper(Environment.getCurrent().getName());

                        try {
                            config = slurper.parse(configURL);
                            ConfigurationHelper.initConfig(config, null, application.getClassLoader());
                            ConfigurationHolder.setConfig(config);
                            configLastModified = lastModified;
                            application.configChanged();
                            informPluginsOfConfigChange();
                        }
                        catch (GroovyRuntimeException gre) {
                            LOG.error("Unable to reload configuration. Please correct problem and try again: " + gre.getMessage(), gre);
{'fraction_non_alphanumeric': 0.09180559690384042, 'fraction_numerical': 0.0012280440607323607, 'mean_word_length': 2.3473467862481314, 'pattern_counts': {'":': 0, '<': 147, '<?xml version=': 0, '>': 141, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4047163', 'n_tokens_mistral': 14071, 'n_tokens_neox': 13373, 'n_words': 3248}