<commit_before>package com.redhat.ceylon.compiler.typechecker.analyzer;

import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import com.redhat.ceylon.cmr.api.ArtifactContext;
import com.redhat.ceylon.cmr.api.ArtifactResult;
import com.redhat.ceylon.cmr.api.RepositoryManager;
import com.redhat.ceylon.compiler.typechecker.context.Context;
import com.redhat.ceylon.compiler.typechecker.context.PhasedUnit;
import com.redhat.ceylon.compiler.typechecker.context.PhasedUnits;
import com.redhat.ceylon.compiler.typechecker.model.Module;
import com.redhat.ceylon.compiler.typechecker.model.ModuleImport;

/**
 * Validate module dependency:
 *  - make sure all modules are available
 *  - get modules from local or remote repos if necessary
 *  - parse and process external modules
 *
 * @author <NAME> <<EMAIL>>
 */
public class ModuleValidator {
    private final Context context;
    private List<PhasedUnits> phasedUnitsOfDependencies;
    private final ModuleManager moduleManager;

    
    public ModuleValidator(Context context, PhasedUnits phasedUnits) {
        this.context = context;
        this.moduleManager = phasedUnits.getModuleManager();
    }

    public List<PhasedUnits> getPhasedUnitsOfDependencies() {
        return phasedUnitsOfDependencies;
    }

    /**
     *  At this stage we need to
     *  - resolve all non local modules (recursively)
     *  - build the object model of these compiled modules
     *  - declare a missing module as an error
     *  - detect circular dependencies
     *  - detect module version conflicts
     */
    public void verifyModuleDependencyTree() {
        phasedUnitsOfDependencies = new ArrayList<PhasedUnits>();
        LinkedList<Module> dependencyTree = new LinkedList<Module>();
        // only verify modules we compile (and default/language), as that makes us traverse their dependencies anyways
        Set<Module> compiledModules = moduleManager.getCompiledModules();
        List<Module> modules = new ArrayList<Module>(compiledModules.size()+2);
        modules.addAll(compiledModules);
        modules.add(context.getModules().getDefaultModule());
        modules.add(context.getModules().getLanguageModule());
        for (Module module : modules) {
            dependencyTree.addLast(module);
            //we don't care about propagated dependency here as top modules are independent from one another
            verifyModuleDependencyTree(module.getImports(), dependencyTree, new ArrayList<Module>(), ImportDepth.First);
            dependencyTree.pollLast();
        }
        moduleManager.addImplicitImports();
        executeExternalModulePhases();
    }

    /**
     * Used to represent import links with compiled modules
     */
    private enum ImportDepth {
        /**
         * Represents a module directly imported by a compiled module
         */
        First {
            @Override
            public ImportDepth forModuleImport(ModuleImport moduleImport) {
                return Required;
            }

            @Override
            public boolean isVisibleToCompiledModules() {
                return true;
            }
        }, 
        /**
         * Represents a module indirectly imported by a compiled module, through a chain of exported/shared
         * modules imported by a module imported directly by a compiled module
         */
        Required {
            @Override
            public ImportDepth forModuleImport(ModuleImport moduleImport) {
                return moduleImport.isExport() ? Required : Transitive;
            }

            @Override
            public boolean isVisibleToCompiledModules() {
                return true;
            }
        }, 
        /**
         * Represents a module indirectly imported by a compiled module, but not visible to it because
         * it was not shared/exported to it
         */
        Transitive {
            @Override
            public ImportDepth forModuleImport(ModuleImport moduleImport) {
                return Transitive;
            }

            @Override
            public boolean isVisibleToCompiledModules() {
                return false;
            }
        };

        /**
         * Returns a new ImportDepth for the given module import
         */
        public abstract ImportDepth forModuleImport(ModuleImport moduleImport);

        /**
         * Returns true if this import is visible to the compiled modules
         */
        public abstract boolean isVisibleToCompiledModules();
    }
    
    private void verifyModuleDependencyTree(
            Collection<ModuleImport> moduleImports,
            LinkedList<Module> dependencyTree,
            List<Module> propagatedDependencies, 
            ImportDepth importDepth) {
        List<Module> visibleDependencies = new ArrayList<Module>();
        visibleDependencies.add(dependencyTree.getLast()); //first addition => no possible conflict
        for (ModuleImport moduleImport : moduleImports) {
            Module module = moduleImport.getModule();
            if (moduleManager.findModule(module, dependencyTree, true) != null) {
                //circular dependency: stop right here
                return;
            }
            Iterable<String> searchedArtifactExtensions = moduleManager.getSearchedArtifactExtensions();
            ImportDepth newImportDepth = importDepth.forModuleImport(moduleImport);
            
            if ( ! module.isAvailable() ) {
                //try and load the module from the repository
                ArtifactResult artifact = null;
                RepositoryManager repositoryManager = context.getRepositoryManager();
                Exception exceptionOnGetArtifact = null;
                ArtifactContext artifactContext = null;
                for(String extension : searchedArtifactExtensions){
                    artifactContext = new ArtifactContext(module.getNameAsString(), module.getVersion(), "."+extension);
                    try {
                        artifact = repositoryManager.getArtifactResult(artifactContext);
                        if(artifact != null) 
                            break;
                    } catch (Exception e) {
                        exceptionOnGetArtifact = e;
                    }
                }
                if (artifact == null) {
                    //not there => error
                    ModuleHelper.buildErrorOnMissingArtifact(artifactContext, module, moduleImport, dependencyTree, exceptionOnGetArtifact, moduleManager);
                }
                else {
                    //parse module units and build module dependency and carry on
                    boolean forCompiledModule = newImportDepth.isVisibleToCompiledModules();
                    moduleManager.resolveModule(artifact, module, moduleImport, dependencyTree, phasedUnitsOfDependencies, forCompiledModule);
                }
            }
            dependencyTree.addLast(module);
            List<Module> subModulePropagatedDependencies = new ArrayList<Module>();
            verifyModuleDependencyTree( module.getImports(), dependencyTree, subModulePropagatedDependencies, newImportDepth );
            //visible dependency += subModule + subModulePropagatedDependencies
            checkAndAddDependency(visibleDependencies, module, dependencyTree);
            for (Module submodule : subModulePropagatedDependencies) {
                checkAndAddDependency(visibleDependencies, submodule, dependencyTree);
            }
            //propagated dependency += if subModule.export then subModule + subModulePropagatedDependencies
            if (moduleImport.isExport()) {
                checkAndAddDependency(propagatedDependencies, module, dependencyTree);
                for (Module submodule : subModulePropagatedDependencies) {
                    checkAndAddDependency(propagatedDependencies, submodule, dependencyTree);
                }
            }
            dependencyTree.pollLast();
        }
    }

    private void checkAndAddDependency(List<Module> dependencies, Module module, LinkedList<Module> dependencyTree) {
        Module dupe = moduleManager.findModule(module, dependencies, false);
        if (dupe != null && !isSameVersion(module, dupe)) {<commit_msg>Fixed ceylon/ceylon-module-resolver#69 Network request for HTTP/HEAD on cached modules<commit_after>package com.redhat.ceylon.compiler.typechecker.analyzer;

import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import com.redhat.ceylon.cmr.api.ArtifactContext;
import com.redhat.ceylon.cmr.api.ArtifactResult;
import com.redhat.ceylon.cmr.api.RepositoryManager;
import com.redhat.ceylon.compiler.typechecker.context.Context;
import com.redhat.ceylon.compiler.typechecker.context.PhasedUnit;
import com.redhat.ceylon.compiler.typechecker.context.PhasedUnits;
import com.redhat.ceylon.compiler.typechecker.model.Module;
import com.redhat.ceylon.compiler.typechecker.model.ModuleImport;

/**
 * Validate module dependency:
 *  - make sure all modules are available
 *  - get modules from local or remote repos if necessary
 *  - parse and process external modules
 *
 * @author <NAME> <<EMAIL>>
 */
public class ModuleValidator {
    private final Context context;
    private List<PhasedUnits> phasedUnitsOfDependencies;
    private final ModuleManager moduleManager;

    
    public ModuleValidator(Context context, PhasedUnits phasedUnits) {
        this.context = context;
        this.moduleManager = phasedUnits.getModuleManager();
    }

    public List<PhasedUnits> getPhasedUnitsOfDependencies() {
        return phasedUnitsOfDependencies;
    }

    /**
     *  At this stage we need to
     *  - resolve all non local modules (recursively)
     *  - build the object model of these compiled modules
     *  - declare a missing module as an error
     *  - detect circular dependencies
     *  - detect module version conflicts
     */
    public void verifyModuleDependencyTree() {
        phasedUnitsOfDependencies = new ArrayList<PhasedUnits>();
        LinkedList<Module> dependencyTree = new LinkedList<Module>();
        // only verify modules we compile (and default/language), as that makes us traverse their dependencies anyways
        Set<Module> compiledModules = moduleManager.getCompiledModules();
        List<Module> modules = new ArrayList<Module>(compiledModules.size()+2);
        modules.addAll(compiledModules);
        modules.add(context.getModules().getDefaultModule());
        modules.add(context.getModules().getLanguageModule());
        for (Module module : modules) {
            dependencyTree.addLast(module);
            //we don't care about propagated dependency here as top modules are independent from one another
            verifyModuleDependencyTree(module.getImports(), dependencyTree, new ArrayList<Module>(), ImportDepth.First);
            dependencyTree.pollLast();
        }
        moduleManager.addImplicitImports();
        executeExternalModulePhases();
    }

    /**
     * Used to represent import links with compiled modules
     */
    private enum ImportDepth {
        /**
         * Represents a module directly imported by a compiled module
         */
        First {
            @Override
            public ImportDepth forModuleImport(ModuleImport moduleImport) {
                return Required;
            }

            @Override
            public boolean isVisibleToCompiledModules() {
                return true;
            }
        }, 
        /**
         * Represents a module indirectly imported by a compiled module, through a chain of exported/shared
         * modules imported by a module imported directly by a compiled module
         */
        Required {
            @Override
            public ImportDepth forModuleImport(ModuleImport moduleImport) {
                return moduleImport.isExport() ? Required : Transitive;
            }

            @Override
            public boolean isVisibleToCompiledModules() {
                return true;
            }
        }, 
        /**
         * Represents a module indirectly imported by a compiled module, but not visible to it because
         * it was not shared/exported to it
         */
        Transitive {
            @Override
            public ImportDepth forModuleImport(ModuleImport moduleImport) {
                return Transitive;
            }

            @Override
            public boolean isVisibleToCompiledModules() {
                return false;
            }
        };

        /**
         * Returns a new ImportDepth for the given module import
         */
        public abstract ImportDepth forModuleImport(ModuleImport moduleImport);

        /**
         * Returns true if this import is visible to the compiled modules
         */
        public abstract boolean isVisibleToCompiledModules();
    }
    
    private void verifyModuleDependencyTree(
            Collection<ModuleImport> moduleImports,
            LinkedList<Module> dependencyTree,
            List<Module> propagatedDependencies, 
            ImportDepth importDepth) {
        List<Module> visibleDependencies = new ArrayList<Module>();
        visibleDependencies.add(dependencyTree.getLast()); //first addition => no possible conflict
        for (ModuleImport moduleImport : moduleImports) {
            Module module = moduleImport.getModule();
            if (moduleManager.findModule(module, dependencyTree, true) != null) {
                //circular dependency: stop right here
                return;
            }
            Iterable<String> searchedArtifactExtensions = moduleManager.getSearchedArtifactExtensions();
            ImportDepth newImportDepth = importDepth.forModuleImport(moduleImport);
            
            if ( ! module.isAvailable() ) {
                //try and load the module from the repository
                ArtifactResult artifact = null;
                RepositoryManager repositoryManager = context.getRepositoryManager();
                Exception exceptionOnGetArtifact = null;
                ArtifactContext artifactContext = new ArtifactContext(module.getNameAsString(), module.getVersion(), getArtifactSuffixes(searchedArtifactExtensions));
                try {
                    artifact = repositoryManager.getArtifactResult(artifactContext);
                } catch (Exception e) {
                    exceptionOnGetArtifact = e;
                }
                if (artifact == null) {
                    //not there => error
                    ModuleHelper.buildErrorOnMissingArtifact(artifactContext, module, moduleImport, dependencyTree, exceptionOnGetArtifact, moduleManager);
                }
                else {
                    //parse module units and build module dependency and carry on
                    boolean forCompiledModule = newImportDepth.isVisibleToCompiledModules();
                    moduleManager.resolveModule(artifact, module, moduleImport, dependencyTree, phasedUnitsOfDependencies, forCompiledModule);
                }
            }
            dependencyTree.addLast(module);
            List<Module> subModulePropagatedDependencies = new ArrayList<Module>();
            verifyModuleDependencyTree( module.getImports(), dependencyTree, subModulePropagatedDependencies, newImportDepth );
            //visible dependency += subModule + subModulePropagatedDependencies
            checkAndAddDependency(visibleDependencies, module, dependencyTree);
            for (Module submodule : subModulePropagatedDependencies) {
                checkAndAddDependency(visibleDependencies, submodule, dependencyTree);
            }
            //propagated dependency += if subModule.export then subModule + subModulePropagatedDependencies
            if (moduleImport.isExport()) {
                checkAndAddDependency(propagatedDependencies, module, dependencyTree);
                for (Module submodule : subModulePropagatedDependencies) {
                    checkAndAddDependency(propagatedDependencies, submodule, dependencyTree);
                }
            }
            dependencyTree.pollLast();
        }
    }

    private String[] getArtifactSuffixes(Iterable<String> extensions) {
        ArrayList<String> suffixes = new ArrayList<String>();
        for (String ext : extensions) {
            suffixes.add("." + ext);
        }
        return suffixes.toArray(new String[suffixes.size()]);
    }
    
    private void checkAndAddDependency(List<Module> dependencies, Module module, LinkedList<Module> dependencyTree) {
        Module dupe = moduleManager.findModule(module, dependencies, false);
        if (dupe != null && !isSameVersion(module, dupe)) {
{'fraction_non_alphanumeric': 0.07478581627796288, 'fraction_numerical': 0.00023798191337458352, 'mean_word_length': 2.3577706751897725, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '377289', 'n_tokens_mistral': 3949, 'n_tokens_neox': 3815, 'n_words': 1013}