<commit_before>package de.japkit.processor

import de.japkit.annotations.Behavior
import de.japkit.el.ELSupport
import de.japkit.metaannotations.Trigger
import de.japkit.model.GenTypeElement
import de.japkit.rules.AnnotationExtensions
import de.japkit.rules.RuleFactory
import de.japkit.rules.TriggerAnnotationRule
import de.japkit.services.ElementsExtensions
import de.japkit.services.ExtensionRegistry
import de.japkit.services.GenerateClassContext
import de.japkit.services.MessageCollector
import de.japkit.services.TypeElementNotFoundException
import de.japkit.services.TypesExtensions
import de.japkit.services.TypesRegistry
import java.util.Collections
import java.util.HashMap
import java.util.HashSet
import java.util.List
import java.util.Map
import java.util.Set
import javax.annotation.processing.AbstractProcessor
import javax.annotation.processing.ProcessingEnvironment
import javax.annotation.processing.RoundEnvironment
import javax.annotation.processing.SupportedSourceVersion
import javax.lang.model.SourceVersion
import javax.lang.model.element.AnnotationMirror
import javax.lang.model.element.TypeElement
import javax.lang.model.util.Elements
import javax.lang.model.util.Types
import javax.tools.Diagnostic.Kind

import static extension de.japkit.util.MoreCollectionExtensions.*
import de.japkit.annotations.RuntimeMetadata

@SupportedSourceVersion(SourceVersion.RELEASE_6)
/**
 * Processes all annotations annotated with @GenerateClass.
 */
class JapkitProcessor extends AbstractProcessor {

	extension ProcessingEnvironment processingEnv
	protected Elements elementUtils
	protected Types typeUtils

	extension ElementsExtensions elementsExtensions
	extension TypesExtensions typesExtensions

	extension AnnotationExtensions annotationExtensions

	extension MessageCollector messageCollector
	extension GenerateClassContext generateClassContext
	extension TypesRegistry typesRegistry
	extension RuleFactory ruleFactory
	extension ELSupport elSupport

	//annotated classes that have to be re-considered in a later round
	val Map<String, TypeElementNotFoundException> deferredClasses = new HashMap

	override init(ProcessingEnvironment processingEnv) {

		super.init(processingEnv)
		this.processingEnv = processingEnv;
		this.elementUtils = processingEnv.elementUtils
		this.typeUtils = processingEnv.typeUtils
		ExtensionRegistry.cleanup //Make sure old stuff is removed.
		ExtensionRegistry.register(ProcessingEnvironment, processingEnv)
		ExtensionRegistry.register(Types, typeUtils)
		ExtensionRegistry.register(Elements, elementUtils)

		this.elementsExtensions = ExtensionRegistry.get(ElementsExtensions);
		this.typesExtensions = ExtensionRegistry.get(TypesExtensions);

		//this.formatter = new EclipseFormatter(messager)
		annotationExtensions = ExtensionRegistry.get(AnnotationExtensions)
		messageCollector = ExtensionRegistry.get(MessageCollector)
		messageCollector.diagnosticLogging = "true".equals(processingEnv.options.get("diagnosticMessages"))
		
		printDiagnosticMessage(['''Init japkit annotation processor.''']);

		generateClassContext = ExtensionRegistry.get(GenerateClassContext)

		typesRegistry = ExtensionRegistry.get(TypesRegistry)
		ruleFactory = ExtensionRegistry.get(RuleFactory)
		elSupport = ExtensionRegistry.get(ELSupport)
		

	}

	override finalize() {

		//Argh. Can we do this in final round instead? Leider nicht so recht... Beim inkrementellen Build in Eclipse kommt nach der letzten Runde 
		//meist nochmal eine neue Runde (WTF !?!) und da wir der gleiche anno-processor verwendet.
		ExtensionRegistry.cleanup
	}

	override getSupportedAnnotationTypes() {
		val set = newHashSet('''«Behavior.package.name».*''','''«Trigger.package.name».*''')
		val annotationsOption = processingEnv.options.get("annotations")
		if (annotationsOption != null) {
			annotationsOption.split(",").forEach[set.add(it)]
		} else {
			messager.printMessage(Kind.ERROR, "The processor option 'annotations' is not set. ");
		}
		set
	}

	override getSupportedOptions() {
		#{"annotations", "diagnosticMessages", "templateDir"}
	}

	override process(Set<? extends TypeElement> annotations, extension RoundEnvironment roundEnv) {
		elementsExtensions.clearCaches
		//The rules cannot be reused in later rounds since they contain references to elements that are no longer valid.
		ruleFactory.clearCaches
		typesRegistry.clearCaches

		val startTime = System.currentTimeMillis

		printDiagnosticMessage(
			[
				'''New Round. «roundEnv.rootElements» Annotations: «annotations» errorRaised «roundEnv.errorRaised» processingOver: «roundEnv.
					processingOver»'''])

		if (roundEnv.processingOver) {
			typesRegistry.persist

			//Do not generate anything in last round but just report all errors that have been collected so far.
			printAllErrors()
			return false
		}

		

		//The annotated classes that have been defered by previous rounds. During the round, some of them will be removed and processed.
		//But also, new ones might be added
		val annotatedClassesToDefer = new HashSet(deferredClasses.keySet.map[getTypeElement].toSet)
		deferredClasses.clear

		val classesToProcessUnfiltered = roundEnv.rootElements.typeElements.toList
		val classesToProcess = new HashSet(classesToProcessUnfiltered)

		//Search for trigger annotations
		val annotatedClassesAndTriggerAnnotations = classesToProcess.toInvertedMap[triggerAnnotationsAndShadowFlag].filter[ac, t|
			!t.empty]
			
		//type elements that ARE trigger annotations
		val triggerAnnotations = new HashSet(classesToProcess.filter[triggerAnnotation].map[it.qualifiedName.toString].toSet)
		triggerAnnotations.addAll(getTriggerAnnotationsForMetaTypeElements(classesToProcess))
		
		val classesWithTrigger = triggerAnnotations.map[findAllTypeElementsWithTriggerAnnotation(it, false)].flatten.toSet

		//Register all classes with trigger annotations, including the ones with shadow annotations
		annotatedClassesAndTriggerAnnotations.forEach[ac, t|typesRegistry.registerAnnotatedClass(ac, t)]<commit_msg>Cleanup in last processing round instead of in finalize()<commit_after>package de.japkit.processor

import de.japkit.annotations.Behavior
import de.japkit.el.ELSupport
import de.japkit.metaannotations.Trigger
import de.japkit.model.GenTypeElement
import de.japkit.rules.AnnotationExtensions
import de.japkit.rules.RuleFactory
import de.japkit.rules.TriggerAnnotationRule
import de.japkit.services.ElementsExtensions
import de.japkit.services.ExtensionRegistry
import de.japkit.services.GenerateClassContext
import de.japkit.services.MessageCollector
import de.japkit.services.TypeElementNotFoundException
import de.japkit.services.TypesExtensions
import de.japkit.services.TypesRegistry
import java.util.Collections
import java.util.HashMap
import java.util.HashSet
import java.util.List
import java.util.Map
import java.util.Set
import javax.annotation.processing.AbstractProcessor
import javax.annotation.processing.ProcessingEnvironment
import javax.annotation.processing.RoundEnvironment
import javax.annotation.processing.SupportedSourceVersion
import javax.lang.model.SourceVersion
import javax.lang.model.element.AnnotationMirror
import javax.lang.model.element.TypeElement
import javax.lang.model.util.Elements
import javax.lang.model.util.Types
import javax.tools.Diagnostic.Kind

import static extension de.japkit.util.MoreCollectionExtensions.*
import de.japkit.annotations.RuntimeMetadata

@SupportedSourceVersion(SourceVersion.RELEASE_6)
/**
 * Processes all annotations annotated with @GenerateClass.
 */
class JapkitProcessor extends AbstractProcessor {

	extension ProcessingEnvironment processingEnv
	protected Elements elementUtils
	protected Types typeUtils

	extension ElementsExtensions elementsExtensions
	extension TypesExtensions typesExtensions

	extension AnnotationExtensions annotationExtensions

	extension MessageCollector messageCollector
	extension GenerateClassContext generateClassContext
	extension TypesRegistry typesRegistry
	extension RuleFactory ruleFactory
	extension ELSupport elSupport

	//annotated classes that have to be re-considered in a later round
	val Map<String, TypeElementNotFoundException> deferredClasses = new HashMap
	
	boolean servicesInitialized;

	override init(ProcessingEnvironment processingEnv) {

		super.init(processingEnv)
		this.processingEnv = processingEnv;
		this.elementUtils = processingEnv.elementUtils
		this.typeUtils = processingEnv.typeUtils
		
		initServices();

	}
	
	def initServices() {
		if(!servicesInitialized){
			ExtensionRegistry.cleanup //Make sure old stuff is removed.
			ExtensionRegistry.register(ProcessingEnvironment, this.processingEnv)
			ExtensionRegistry.register(Types, typeUtils)
			ExtensionRegistry.register(Elements, elementUtils)
	
			this.elementsExtensions = ExtensionRegistry.get(ElementsExtensions);
			this.typesExtensions = ExtensionRegistry.get(TypesExtensions);
	
			annotationExtensions = ExtensionRegistry.get(AnnotationExtensions)
			messageCollector = ExtensionRegistry.get(MessageCollector)
			messageCollector.diagnosticLogging = "true".equals(this.processingEnv.options.get("diagnosticMessages"))
			
			printDiagnosticMessage(['''Init japkit annotation processor.''']);
	
			generateClassContext = ExtensionRegistry.get(GenerateClassContext)
	
			typesRegistry = ExtensionRegistry.get(TypesRegistry)
			ruleFactory = ExtensionRegistry.get(RuleFactory)
			elSupport = ExtensionRegistry.get(ELSupport)	
			servicesInitialized = true;	
		}
	}
	
	def cleanupServices() {
		ExtensionRegistry.cleanup;
		servicesInitialized = false;	
	}

	override getSupportedAnnotationTypes() {
		val set = newHashSet('''«Behavior.package.name».*''','''«Trigger.package.name».*''')
		val annotationsOption = processingEnv.options.get("annotations")
		if (annotationsOption != null) {
			annotationsOption.split(",").forEach[set.add(it)]
		} else {
			messager.printMessage(Kind.ERROR, "The processor option 'annotations' is not set. ");
		}
		set
	}

	override getSupportedOptions() {
		#{"annotations", "diagnosticMessages", "templateDir"}
	}

	override process(Set<? extends TypeElement> annotations, extension RoundEnvironment roundEnv) {
		//Necessary here, since we cleanup in the last round, but in Eclipse, there are multiple "last rounds" during incremental build.
		initServices();
		
		elementsExtensions.clearCaches
		//The rules cannot be reused in later rounds since they contain references to elements that are no longer valid.
		ruleFactory.clearCaches
		typesRegistry.clearCaches

		val startTime = System.currentTimeMillis

		printDiagnosticMessage(
			[
				'''New Round. «roundEnv.rootElements» Annotations: «annotations» errorRaised «roundEnv.errorRaised» processingOver: «roundEnv.
					processingOver»'''])

		if (roundEnv.processingOver) {
			typesRegistry.persist

			//Do not generate anything in last round but just report all errors that have been collected so far.
			printAllErrors()
			cleanupServices()
			return false
		}

		

		//The annotated classes that have been defered by previous rounds. During the round, some of them will be removed and processed.
		//But also, new ones might be added
		val annotatedClassesToDefer = new HashSet(deferredClasses.keySet.map[getTypeElement].toSet)
		deferredClasses.clear

		val classesToProcessUnfiltered = roundEnv.rootElements.typeElements.toList
		val classesToProcess = new HashSet(classesToProcessUnfiltered)

		//Search for trigger annotations
		val annotatedClassesAndTriggerAnnotations = classesToProcess.toInvertedMap[triggerAnnotationsAndShadowFlag].filter[ac, t|
			!t.empty]
			
		//type elements that ARE trigger annotations
		val triggerAnnotations = new HashSet(classesToProcess.filter[triggerAnnotation].map[it.qualifiedName.toString].toSet)
		triggerAnnotations.addAll(getTriggerAnnotationsForMetaTypeElements(classesToProcess))
		
		val classesWithTrigger = triggerAnnotations.map[findAllTypeElementsWithTriggerAnnotation(it, false)].flatten.toSet

		//Register all classes with trigger annotations, including the ones with shadow annotations
		annotatedClassesAndTriggerAnnotations.forEach[ac, t|typesRegistry.registerAnnotatedClass(ac, t)]
{'fraction_non_alphanumeric': 0.07575632676613635, 'fraction_numerical': 0.000164866870002473, 'mean_word_length': 5.980437284234752, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1120000', 'n_tokens_mistral': 3384, 'n_tokens_neox': 3069, 'n_words': 810}