<commit_before>/*
 * Copyright Red Hat Inc. and/or its affiliates and other contributors
 * as indicated by the authors tag. All rights reserved.
 *
 * This copyrighted material is made available to anyone wishing to use,
 * modify, copy, or redistribute it subject to the terms and conditions
 * of the GNU General Public License version 2.
 * 
 * This particular file is subject to the "Classpath" exception as provided in the 
 * LICENSE file that accompanied this code.
 * 
 * This program is distributed in the hope that it will be useful, but WITHOUT A
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 * PARTICULAR PURPOSE.  See the GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License,
 * along with this distribution; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 * MA  02110-1301, USA.
 */
package org.eclipse.ceylon.compiler.java.test;

import java.io.File;
import java.io.FilenameFilter;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.eclipse.ceylon.common.FileUtil;
import org.eclipse.ceylon.compiler.java.codegen.Decl;
import org.eclipse.ceylon.compiler.java.tools.CeylonLog;
import org.eclipse.ceylon.compiler.java.tools.CeyloncFileManager;
import org.eclipse.ceylon.compiler.java.tools.LanguageCompiler;
import org.junit.Assert;
import org.junit.runner.Description;
import org.junit.runner.Runner;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunNotifier;
import org.junit.runners.ParentRunner;
import org.junit.runners.model.InitializationError;

import org.eclipse.ceylon.cmr.api.ArtifactContext;
import org.eclipse.ceylon.compiler.java.runtime.metamodel.Metamodel;
import org.eclipse.ceylon.compiler.typechecker.TypeChecker;
import org.eclipse.ceylon.compiler.typechecker.context.PhasedUnit;
import org.eclipse.ceylon.compiler.typechecker.context.PhasedUnits;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.AnyClass;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.CompilationUnit;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.CompilerAnnotation;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.Declaration;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.Identifier;
import org.eclipse.ceylon.compiler.typechecker.tree.Visitor;
import org.eclipse.ceylon.javax.tools.Diagnostic.Kind;
import org.eclipse.ceylon.javax.tools.DiagnosticListener;
import org.eclipse.ceylon.langtools.tools.javac.main.Main.Result;
import org.eclipse.ceylon.langtools.tools.javac.util.Context;
import org.eclipse.ceylon.model.cmr.JDKUtils;
import org.eclipse.ceylon.model.cmr.JDKUtils.JDK;
import org.eclipse.ceylon.model.loader.AbstractModelLoader;
import org.eclipse.ceylon.model.typechecker.model.ModelUtil;
import org.eclipse.ceylon.model.typechecker.model.Module;
import org.eclipse.ceylon.tools.classpath.CeylonClasspathTool;

/**
 * JUnit Runner for a Ceylon module.
 * 
 * <ul>
 * <li>Works on a class annotated with {@link TestModule} and 
 * {@link org.junit.runner.RunWith RunWith(CeylonModuleRunner.class)} in the 
 * root package of the module to be tested.</li>
 * <li>The top-level classes are
 * scanned for {@code shared} methods with the {@code @test} compiler 
 * annotation.</li>
 * <li>The module is compiled</li>
 * <li>The test classes are loaded. Any which can't be loaded are treated as
 * test failures</li>
 * <li>The test methods on the test classes are executed (by the {@link CeylonTestRunner}</li>
 * <li>Exceptions thrown by the test methods are treated as failures</li>
 * </ol> 
 * @author tom
 */
public class CeylonModuleRunner extends ParentRunner<Runner> {

    private HashMap<Runner, Description> children;

    private boolean errorIfNoTests;
    
    private File outRepo;
    
    private TestLoader testLoader;

    public CeylonModuleRunner(Class<?> moduleSuiteClass) throws InitializationError {
        super(moduleSuiteClass);
        try {
            TestModule testModule = getTestModuleAnno(moduleSuiteClass);
            this.errorIfNoTests = testModule.errorIfNoTests();
            this.testLoader = testModule.testLoader().newInstance();
            this.children = new LinkedHashMap<Runner, Description>();
            
            File srcDir = new File(testModule.srcDirectory());
            File resDir = new File(testModule.resDirectory());
            outRepo = Files.createTempDirectory("ceylon-module-runner-").toFile();
            
            String[] modules = testModule.modules();
            if(modules.length == 0)
                modules = new String[]{ moduleSuiteClass.getPackage().getName() };
            
            Set<String> removeAtRuntime = new HashSet<String>();
            Collections.addAll(removeAtRuntime, testModule.removeAtRuntime());
            if(compileAndRun(srcDir, resDir, outRepo, modules, testModule.dependencies(), 
                          testModule.options(), removeAtRuntime, 
                          testModule.modulesUsingCheckFunction(),
                          testModule.modulesUsingCheckModule())){

                for(ModuleSpecifier module : testModule.runModulesInNewJvm()){
                    makeModuleRunnerInNewJvm(module);
                }
            }
        } catch (RuntimeException e) {
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    private void makeModuleRunnerInNewJvm(final ModuleSpecifier module) {
        final Description description = Description.createTestDescription(getClass(), "Run "+module.module()+" in new JVM");
        
        Runner runner = new Runner(){
            @Override
            public Description getDescription() {
                return description;
            }

            @Override
            public void run(RunNotifier notifier) {
                notifier.fireTestStarted(description);
                try{
                    String moduleName = module.module();
                    String version = Module.DEFAULT_MODULE_NAME.equals(moduleName) ? null : module.version();
                    String runClass = module.runClass();
                    if(runClass.isEmpty())
                        runClass = moduleName + ".run_";<commit_msg>Propagate error properly in module runner tests<commit_after>/*
 * Copyright Red Hat Inc. and/or its affiliates and other contributors
 * as indicated by the authors tag. All rights reserved.
 *
 * This copyrighted material is made available to anyone wishing to use,
 * modify, copy, or redistribute it subject to the terms and conditions
 * of the GNU General Public License version 2.
 * 
 * This particular file is subject to the "Classpath" exception as provided in the 
 * LICENSE file that accompanied this code.
 * 
 * This program is distributed in the hope that it will be useful, but WITHOUT A
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 * PARTICULAR PURPOSE.  See the GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License,
 * along with this distribution; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 * MA  02110-1301, USA.
 */
package org.eclipse.ceylon.compiler.java.test;

import java.io.File;
import java.io.FilenameFilter;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.eclipse.ceylon.common.FileUtil;
import org.eclipse.ceylon.compiler.java.codegen.Decl;
import org.eclipse.ceylon.compiler.java.tools.CeylonLog;
import org.eclipse.ceylon.compiler.java.tools.CeyloncFileManager;
import org.eclipse.ceylon.compiler.java.tools.LanguageCompiler;
import org.junit.Assert;
import org.junit.runner.Description;
import org.junit.runner.Runner;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunNotifier;
import org.junit.runners.ParentRunner;
import org.junit.runners.model.InitializationError;

import org.eclipse.ceylon.cmr.api.ArtifactContext;
import org.eclipse.ceylon.compiler.java.runtime.metamodel.Metamodel;
import org.eclipse.ceylon.compiler.typechecker.TypeChecker;
import org.eclipse.ceylon.compiler.typechecker.context.PhasedUnit;
import org.eclipse.ceylon.compiler.typechecker.context.PhasedUnits;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.AnyClass;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.CompilationUnit;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.CompilerAnnotation;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.Declaration;
import org.eclipse.ceylon.compiler.typechecker.tree.Tree.Identifier;
import org.eclipse.ceylon.compiler.typechecker.tree.Visitor;
import org.eclipse.ceylon.javax.tools.Diagnostic.Kind;
import org.eclipse.ceylon.javax.tools.DiagnosticListener;
import org.eclipse.ceylon.langtools.tools.javac.main.Main.Result;
import org.eclipse.ceylon.langtools.tools.javac.util.Context;
import org.eclipse.ceylon.model.cmr.JDKUtils;
import org.eclipse.ceylon.model.cmr.JDKUtils.JDK;
import org.eclipse.ceylon.model.loader.AbstractModelLoader;
import org.eclipse.ceylon.model.typechecker.model.ModelUtil;
import org.eclipse.ceylon.model.typechecker.model.Module;
import org.eclipse.ceylon.tools.classpath.CeylonClasspathTool;

/**
 * JUnit Runner for a Ceylon module.
 * 
 * <ul>
 * <li>Works on a class annotated with {@link TestModule} and 
 * {@link org.junit.runner.RunWith RunWith(CeylonModuleRunner.class)} in the 
 * root package of the module to be tested.</li>
 * <li>The top-level classes are
 * scanned for {@code shared} methods with the {@code @test} compiler 
 * annotation.</li>
 * <li>The module is compiled</li>
 * <li>The test classes are loaded. Any which can't be loaded are treated as
 * test failures</li>
 * <li>The test methods on the test classes are executed (by the {@link CeylonTestRunner}</li>
 * <li>Exceptions thrown by the test methods are treated as failures</li>
 * </ol> 
 * @author tom
 */
public class CeylonModuleRunner extends ParentRunner<Runner> {

    private HashMap<Runner, Description> children;

    private boolean errorIfNoTests;
    
    private File outRepo;
    
    private TestLoader testLoader;

    public CeylonModuleRunner(Class<?> moduleSuiteClass) throws InitializationError {
        super(moduleSuiteClass);
        try {
            TestModule testModule = getTestModuleAnno(moduleSuiteClass);
            this.errorIfNoTests = testModule.errorIfNoTests();
            this.testLoader = testModule.testLoader().newInstance();
            this.children = new LinkedHashMap<Runner, Description>();
            
            File srcDir = new File(testModule.srcDirectory());
            File resDir = new File(testModule.resDirectory());
            outRepo = Files.createTempDirectory("ceylon-module-runner-").toFile();
            
            String[] modules = testModule.modules();
            if(modules.length == 0)
                modules = new String[]{ moduleSuiteClass.getPackage().getName() };
            
            Set<String> removeAtRuntime = new HashSet<String>();
            Collections.addAll(removeAtRuntime, testModule.removeAtRuntime());
            if(compileAndRun(srcDir, resDir, outRepo, modules, testModule.dependencies(), 
                          testModule.options(), removeAtRuntime, 
                          testModule.modulesUsingCheckFunction(),
                          testModule.modulesUsingCheckModule())){

                for(ModuleSpecifier module : testModule.runModulesInNewJvm()){
                    makeModuleRunnerInNewJvm(module);
                }
            }
        } catch (RuntimeException e) {
            throw e;
        } catch (InitializationError e) {
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    private void makeModuleRunnerInNewJvm(final ModuleSpecifier module) {
        final Description description = Description.createTestDescription(getClass(), "Run "+module.module()+" in new JVM");
        
        Runner runner = new Runner(){
            @Override
            public Description getDescription() {
                return description;
            }

            @Override
            public void run(RunNotifier notifier) {
                notifier.fireTestStarted(description);
                try{
                    String moduleName = module.module();
                    String version = Module.DEFAULT_MODULE_NAME.equals(moduleName) ? null : module.version();
                    String runClass = module.runClass();
                    if(runClass.isEmpty())
                        runClass = moduleName + ".run_";
{'fraction_non_alphanumeric': 0.09631132690774694, 'fraction_numerical': 0.0018841945068483223, 'mean_word_length': 3.3206011271133375, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1698542', 'n_tokens_mistral': 3795, 'n_tokens_neox': 3702, 'n_words': 956}