<commit_before>package consulo.webService.update.pluginAnalyzer;

import gnu.trove.THashMap;

import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.jdom.Document;
import org.jdom.Element;
import org.jetbrains.annotations.NotNull;
import org.picocontainer.PicoContainer;
import com.google.common.collect.Lists;
import com.intellij.ide.plugins.IdeaPluginDescriptorImpl;
import com.intellij.lang.Language;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.extensions.PluginId;
import com.intellij.psi.SingleRootFileViewProvider;
import com.intellij.util.PathUtil;
import com.intellij.util.ThrowableConsumer;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.MultiMap;
import com.intellij.util.lang.UrlClassLoader;
import consulo.pluginAnalyzer.Analyzer;
import consulo.webService.ChildService;

/**
 * @author VISTALL
 * @since 20-Sep-16
 */
public class PluginAnalyzerService extends ChildService
{
	private static final Logger LOGGER = Logger.getInstance(PluginAnalyzerService.class);

	private final List<URL> platformClassUrls = new ArrayList<>();

	@Override
	protected void initImpl(File pluginChannelDir)
	{
		// core-api
		addUrlByClass(Language.class);
		// core-impl
		addUrlByClass(SingleRootFileViewProvider.class);
		// platform-api
		addUrlByClass("com.intellij.execution.configurations.ConfigurationType");
		// platform-impl
		addUrlByClass("consulo.extension.impl.ModuleExtensionImpl");
		// extensions
		addUrlByClass(PluginId.class);
		// picocontainer
		addUrlByClass(PicoContainer.class);
		// util
		addUrlByClass(ContainerUtil.class);
		// jdom
		addUrlByClass(Document.class);
		// trove4j
		addUrlByClass(THashMap.class);
		// guava
		addUrlByClass(Lists.class);
		// plugin-analyzer-rt
		addUrlByClass(Analyzer.class);
	}

	private void addUrlByClass(Class<?> clazz)
	{
		addUrlByClass(clazz.getName());
	}

	private void addUrlByClass(String clazzName)
	{
		try
		{
			Class<?> clazz = Class.forName(clazzName);

			String jarPathForClass = PathUtil.getJarPathForClass(clazz);

			platformClassUrls.add(new File(jarPathForClass).toURI().toURL());
		}
		catch(ClassNotFoundException | java.net.MalformedURLException e)
		{
			LOGGER.error("Class " + clazzName + " is not found", e);
		}
	}

	@NotNull
	public MultiMap<String, String> analyze(IdeaPluginDescriptorImpl ideaPluginDescriptor) throws Exception
	{
		List<URL> urls = new ArrayList<>();
		urls.addAll(platformClassUrls);

		for(File file : ideaPluginDescriptor.getClassPath())
		{
			urls.add(file.toURI().toURL());
		}

		UrlClassLoader urlClassLoader = UrlClassLoader.build().urls(urls).useCache(false).get();

		Class<?> analyzerClass = urlClassLoader.loadClass(Analyzer.class.getName());
		analyzerClass.getDeclaredMethod("before").invoke(null);

		MultiMap<String, Element> extensions = ideaPluginDescriptor.getExtensions();
		if(extensions == null)
		{
			return MultiMap.empty();
		}

		MultiMap<String, String> data = new MultiMap<String, String>()
		{
			@NotNull
			@Override
			protected Map<String, Collection<String>> createMap()
			{
				return new TreeMap<>();
			}

			@NotNull
			@Override
			protected Collection<String> createCollection()
			{
				return new TreeSet<>();
			}
		};

		for(Map.Entry<String, Collection<Element>> entry : extensions.entrySet())
		{
			String key = entry.getKey();
			switch(key)
			{
				case "com.intellij.configurationType":
					forEachQuiet(entry, element -> {
						String implementation = element.getAttributeValue("implementation");
						if(implementation != null)
						{
							Class<?> aClass = urlClassLoader.loadClass(implementation);

							Constructor constructorForNew = null;
							Constructor<?>[] declaredConstructors = aClass.getDeclaredConstructors();
							for(Constructor<?> declaredConstructor : declaredConstructors)
							{
								if(declaredConstructor.getParameterCount() == 0)
								{
									declaredConstructor.setAccessible(true);
									constructorForNew = declaredConstructor;
								}
							}

							if(constructorForNew == null)
							{
								return;
							}

							Object configurationType = constructorForNew.newInstance();

							Method getId = aClass.getDeclaredMethod("getId");
							String id = (String) getId.invoke(configurationType);

							data.putValue(key, id);
						}
					});
					break;
				case "com.intellij.fileTypeFactory":
					forEachQuiet(entry, element -> {
						String implementation = element.getAttributeValue("implementation");
						if(implementation != null)
						{
							Class<?> aClass = urlClassLoader.loadClass(implementation);

							Object fileTypeFactory = aClass.newInstance();

							Class<?> fileTypeFactoryClass = Class.forName("com.intellij.openapi.fileTypes.FileTypeFactory", true, urlClassLoader);

							Method analyzeFileType = analyzerClass.getDeclaredMethod("analyzeFileType", Set.class, fileTypeFactoryClass);

							Set<String> ext = new TreeSet<>();

							analyzeFileType.invoke(null, ext, fileTypeFactory);

							if(!ext.isEmpty())
							{
								data.putValues(key, ext);
							}
						}
					});
					break;
				case "com.intellij.moduleExtensionProvider":
					forEachQuiet(entry, element -> {
						String extensionKey = element.getAttributeValue("key");
						if(extensionKey != null)
						{
							data.putValue(key, extensionKey);
						}
					});
					break;
			}
		}

		return data;
	}
<commit_msg>don't ignore all extensions if somebody insert foreign logic (XmlFileTypeFactory:38)<commit_after>package consulo.webService.update.pluginAnalyzer;

import gnu.trove.THashMap;

import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.jdom.Document;
import org.jdom.Element;
import org.jetbrains.annotations.NotNull;
import org.picocontainer.PicoContainer;
import com.google.common.collect.Lists;
import com.intellij.ide.plugins.IdeaPluginDescriptorImpl;
import com.intellij.lang.Language;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.extensions.PluginId;
import com.intellij.psi.SingleRootFileViewProvider;
import com.intellij.util.PathUtil;
import com.intellij.util.ThrowableConsumer;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.MultiMap;
import com.intellij.util.lang.UrlClassLoader;
import consulo.pluginAnalyzer.Analyzer;
import consulo.webService.ChildService;

/**
 * @author VISTALL
 * @since 20-Sep-16
 */
public class PluginAnalyzerService extends ChildService
{
	private static final Logger LOGGER = Logger.getInstance(PluginAnalyzerService.class);

	private final List<URL> platformClassUrls = new ArrayList<>();

	@Override
	protected void initImpl(File pluginChannelDir)
	{
		// core-api
		addUrlByClass(Language.class);
		// core-impl
		addUrlByClass(SingleRootFileViewProvider.class);
		// platform-api
		addUrlByClass("com.intellij.execution.configurations.ConfigurationType");
		// platform-impl
		addUrlByClass("consulo.extension.impl.ModuleExtensionImpl");
		// extensions
		addUrlByClass(PluginId.class);
		// picocontainer
		addUrlByClass(PicoContainer.class);
		// util
		addUrlByClass(ContainerUtil.class);
		// jdom
		addUrlByClass(Document.class);
		// trove4j
		addUrlByClass(THashMap.class);
		// guava
		addUrlByClass(Lists.class);
		// plugin-analyzer-rt
		addUrlByClass(Analyzer.class);
	}

	private void addUrlByClass(Class<?> clazz)
	{
		addUrlByClass(clazz.getName());
	}

	private void addUrlByClass(String clazzName)
	{
		try
		{
			Class<?> clazz = Class.forName(clazzName);

			String jarPathForClass = PathUtil.getJarPathForClass(clazz);

			platformClassUrls.add(new File(jarPathForClass).toURI().toURL());
		}
		catch(ClassNotFoundException | java.net.MalformedURLException e)
		{
			LOGGER.error("Class " + clazzName + " is not found", e);
		}
	}

	@NotNull
	public MultiMap<String, String> analyze(IdeaPluginDescriptorImpl ideaPluginDescriptor) throws Exception
	{
		List<URL> urls = new ArrayList<>();
		urls.addAll(platformClassUrls);

		for(File file : ideaPluginDescriptor.getClassPath())
		{
			urls.add(file.toURI().toURL());
		}

		UrlClassLoader urlClassLoader = UrlClassLoader.build().urls(urls).useCache(false).get();

		Class<?> analyzerClass = urlClassLoader.loadClass(Analyzer.class.getName());
		analyzerClass.getDeclaredMethod("before").invoke(null);

		MultiMap<String, Element> extensions = ideaPluginDescriptor.getExtensions();
		if(extensions == null)
		{
			return MultiMap.empty();
		}

		MultiMap<String, String> data = new MultiMap<String, String>()
		{
			@NotNull
			@Override
			protected Map<String, Collection<String>> createMap()
			{
				return new TreeMap<>();
			}

			@NotNull
			@Override
			protected Collection<String> createCollection()
			{
				return new TreeSet<>();
			}
		};

		for(Map.Entry<String, Collection<Element>> entry : extensions.entrySet())
		{
			String key = entry.getKey();
			switch(key)
			{
				case "com.intellij.configurationType":
					forEachQuiet(entry, element -> {
						String implementation = element.getAttributeValue("implementation");
						if(implementation != null)
						{
							Class<?> aClass = urlClassLoader.loadClass(implementation);

							Constructor constructorForNew = null;
							Constructor<?>[] declaredConstructors = aClass.getDeclaredConstructors();
							for(Constructor<?> declaredConstructor : declaredConstructors)
							{
								if(declaredConstructor.getParameterCount() == 0)
								{
									declaredConstructor.setAccessible(true);
									constructorForNew = declaredConstructor;
								}
							}

							if(constructorForNew == null)
							{
								return;
							}

							Object configurationType = constructorForNew.newInstance();

							Method getId = aClass.getDeclaredMethod("getId");
							String id = (String) getId.invoke(configurationType);

							data.putValue(key, id);
						}
					});
					break;
				case "com.intellij.fileTypeFactory":
					forEachQuiet(entry, element -> {
						String implementation = element.getAttributeValue("implementation");
						if(implementation != null)
						{
							Class<?> aClass = urlClassLoader.loadClass(implementation);

							Object fileTypeFactory = aClass.newInstance();

							Class<?> fileTypeFactoryClass = Class.forName("com.intellij.openapi.fileTypes.FileTypeFactory", true, urlClassLoader);

							Set<String> ext = new TreeSet<>();

							Method analyzeFileType = analyzerClass.getDeclaredMethod("analyzeFileType", Set.class, fileTypeFactoryClass);
							try
							{
								analyzeFileType.invoke(null, ext, fileTypeFactory);
							}
							catch(Throwable e)
							{
								// somebodies can insert some logic in factory (com.intellij.xml.XmlFileTypeFactory:38)
								// it can failed, but - we before logic, extensions can be registered
								//LOGGER.error(e);
							}

							if(!ext.isEmpty())
							{
								data.putValues(key, ext);
							}
						}
					});
					break;
				case "com.intellij.moduleExtensionProvider":
					forEachQuiet(entry, element -> {
						String extensionKey = element.getAttributeValue("key");
						if(extensionKey != null)
						{
							data.putValue(key, extensionKey);
						}
					});
					break;
			}
		}

		return data;
	}

{'fraction_non_alphanumeric': 0.11726969463690018, 'fraction_numerical': 0.0013685741168420152, 'mean_word_length': 3.8454206382096974, 'pattern_counts': {'":': 6, '<': 53, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5567813', 'n_tokens_mistral': 4399, 'n_tokens_neox': 3520, 'n_words': 612}