<commit_before>package org.python.pydev.editor.codecompletion.revisited;

import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import org.eclipse.core.resources.IProject;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.jface.text.IDocument;
import org.python.pydev.core.FindInfo;
import org.python.pydev.core.FullRepIterable;
import org.python.pydev.core.ICodeCompletionASTManager;
import org.python.pydev.core.ICompletionRequest;
import org.python.pydev.core.ICompletionState;
import org.python.pydev.core.IModule;
import org.python.pydev.core.IModulesManager;
import org.python.pydev.core.IPythonNature;
import org.python.pydev.core.IToken;
import org.python.pydev.core.ModulesKey;
import org.python.pydev.core.REF;
import org.python.pydev.core.Tuple;
import org.python.pydev.editor.actions.PyAction;
import org.python.pydev.editor.codecompletion.CompletionRequest;
import org.python.pydev.editor.codecompletion.PyCodeCompletion;
import org.python.pydev.editor.codecompletion.revisited.modules.AbstractModule;
import org.python.pydev.editor.codecompletion.revisited.modules.SourceModule;
import org.python.pydev.editor.codecompletion.revisited.visitors.Definition;
import org.python.pydev.parser.PyParser;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.FunctionDef;
import org.python.pydev.parser.visitors.NodeUtils;

public abstract class AbstractASTManager implements ICodeCompletionASTManager, Serializable {

	public AbstractASTManager(){
	}
	
    /**
     * This is the guy that will handle project things for us
     */
    public IModulesManager modulesManager;
    public IModulesManager getModulesManager(){
        return modulesManager;
    }

    


    /**
     * Set the nature this ast manager works with (if no project is available and a nature is).
     */
    public void setNature(IPythonNature nature){
        getModulesManager().setPythonNature(nature);
    }
    
    public IPythonNature getNature() {
        return getModulesManager().getNature();
    }
    
    
	public abstract void setProject(IProject project, boolean restoreDeltas) ;

	public abstract void rebuildModule(File file, IDocument doc, IProject project, IProgressMonitor monitor, IPythonNature nature) ;

	public abstract void removeModule(File file, IProject project, IProgressMonitor monitor) ;
	

    /**
     * Returns the imports that start with a given string. The comparisson is not case dependent. Passes all the modules in the cache.
     * 
     * @param original is the name of the import module eg. 'from toimport import ' would mean that the original is 'toimport'
     * or something like 'foo.bar' or an empty string (if only 'import').
     * @return a Set with the imports as tuples with the name, the docstring.
     */
    public IToken[] getCompletionsForImport(final String original, ICompletionRequest r) {
        CompletionRequest request = (CompletionRequest) r;
        IPythonNature nature = request.nature;
        
        String relative = null;
        if(request.editorFile != null){
            String moduleName = nature.getAstManager().getModulesManager().resolveModule(REF.getFileAbsolutePath(request.editorFile));
            if(moduleName != null){
                String tail = FullRepIterable.headAndTail(moduleName)[0];
                if(original.length() > 0){
                    relative = tail+"."+original;
                }else{
                    relative = tail;
                }
            }
        }
        
        String absoluteModule = original;
        if (absoluteModule.endsWith(".")) {
            absoluteModule = absoluteModule.substring(0, absoluteModule.length() - 1);
        }
        absoluteModule = absoluteModule.toLowerCase().trim();

        //set to hold the completion (no duplicates allowed).
        Set<IToken> set = new HashSet<IToken>();

        //first we get the imports... that complete for the token.
        getAbsoluteImportTokens(absoluteModule, set, PyCodeCompletion.TYPE_IMPORT, false);

        //Now, if we have an initial module, we have to get the completions
        //for it.
        getTokensForModule(original, nature, absoluteModule, set);

        if(relative != null && relative.equals(absoluteModule) == false){
            getAbsoluteImportTokens(relative, set, PyCodeCompletion.TYPE_RELATIVE_IMPORT, false);
            getTokensForModule(relative, nature, relative, set);
        }
        return (IToken[]) set.toArray(new IToken[0]);
    }

    
    
    /**
     * @param moduleToGetTokensFrom the string that represents the token from where we are getting the imports
     * @param set the set where the tokens should be added
     */
    protected void getAbsoluteImportTokens(String moduleToGetTokensFrom, Set<IToken> set, int type, boolean onlyFilesOnSameLevel) {
        for (Iterator iter = Arrays.asList(modulesManager.getAllModules()).iterator(); iter.hasNext();) {
            ModulesKey key = (ModulesKey) iter.next();

            String element = key.name;
			if (element.toLowerCase().startsWith(moduleToGetTokensFrom)) {
                if(onlyFilesOnSameLevel && key.file != null && key.file.isDirectory()){
                	continue; // we only want those that are in the same directory, and not in other directories...<commit_msg>*** empty log message ***<commit_after>package org.python.pydev.editor.codecompletion.revisited;

import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import org.eclipse.core.resources.IProject;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.jface.text.IDocument;
import org.python.pydev.core.FindInfo;
import org.python.pydev.core.FullRepIterable;
import org.python.pydev.core.ICodeCompletionASTManager;
import org.python.pydev.core.ICompletionRequest;
import org.python.pydev.core.ICompletionState;
import org.python.pydev.core.IModule;
import org.python.pydev.core.IModulesManager;
import org.python.pydev.core.IPythonNature;
import org.python.pydev.core.IToken;
import org.python.pydev.core.ModulesKey;
import org.python.pydev.core.REF;
import org.python.pydev.core.Tuple;
import org.python.pydev.editor.actions.PyAction;
import org.python.pydev.editor.codecompletion.CompletionRequest;
import org.python.pydev.editor.codecompletion.PyCodeCompletion;
import org.python.pydev.editor.codecompletion.revisited.modules.AbstractModule;
import org.python.pydev.editor.codecompletion.revisited.modules.SourceModule;
import org.python.pydev.editor.codecompletion.revisited.visitors.Definition;
import org.python.pydev.parser.PyParser;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.FunctionDef;
import org.python.pydev.parser.visitors.NodeUtils;

public abstract class AbstractASTManager implements ICodeCompletionASTManager, Serializable {

	public AbstractASTManager(){
	}
	
    /**
     * This is the guy that will handle project things for us
     */
    public IModulesManager modulesManager;
    public IModulesManager getModulesManager(){
        return modulesManager;
    }

    


    /**
     * Set the nature this ast manager works with (if no project is available and a nature is).
     */
    public void setNature(IPythonNature nature){
        getModulesManager().setPythonNature(nature);
    }
    
    public IPythonNature getNature() {
        return getModulesManager().getNature();
    }
    
    
	public abstract void setProject(IProject project, boolean restoreDeltas) ;

	public abstract void rebuildModule(File file, IDocument doc, IProject project, IProgressMonitor monitor, IPythonNature nature) ;

	public abstract void removeModule(File file, IProject project, IProgressMonitor monitor) ;
	

    /**
     * Returns the imports that start with a given string. The comparisson is not case dependent. Passes all the modules in the cache.
     * 
     * @param original is the name of the import module eg. 'from toimport import ' would mean that the original is 'toimport'
     * or something like 'foo.bar' or an empty string (if only 'import').
     * @return a Set with the imports as tuples with the name, the docstring.
     */
    public IToken[] getCompletionsForImport(final String original, ICompletionRequest r) {
        CompletionRequest request = (CompletionRequest) r;
        IPythonNature nature = request.nature;
        
        String relative = null;
        if(request.editorFile != null){
            String moduleName = nature.getAstManager().getModulesManager().resolveModule(REF.getFileAbsolutePath(request.editorFile));
            if(moduleName != null){
                String tail = FullRepIterable.headAndTail(moduleName)[0];
                if(original.length() > 0){
                    relative = tail+"."+original;
                }else{
                    relative = tail;
                }
            }
        }
        
        String absoluteModule = original;
        if (absoluteModule.endsWith(".")) {
            absoluteModule = absoluteModule.substring(0, absoluteModule.length() - 1);
        }
        //absoluteModule = absoluteModule.toLowerCase().trim();

        //set to hold the completion (no duplicates allowed).
        Set<IToken> set = new HashSet<IToken>();

        //first we get the imports... that complete for the token.
        getAbsoluteImportTokens(absoluteModule, set, PyCodeCompletion.TYPE_IMPORT, false);

        //Now, if we have an initial module, we have to get the completions
        //for it.
        getTokensForModule(original, nature, absoluteModule, set);

        if(relative != null && relative.equals(absoluteModule) == false){
            getAbsoluteImportTokens(relative, set, PyCodeCompletion.TYPE_RELATIVE_IMPORT, false);
            getTokensForModule(relative, nature, relative, set);
        }
        return (IToken[]) set.toArray(new IToken[0]);
    }

    
    
    /**
     * @param moduleToGetTokensFrom the string that represents the token from where we are getting the imports
     * @param set the set where the tokens should be added
     */
    protected void getAbsoluteImportTokens(String moduleToGetTokensFrom, Set<IToken> set, int type, boolean onlyFilesOnSameLevel) {
        for (Iterator iter = Arrays.asList(modulesManager.getAllModules()).iterator(); iter.hasNext();) {
            ModulesKey key = (ModulesKey) iter.next();

            String element = key.name;
			if (element.startsWith(moduleToGetTokensFrom)) {
                if(onlyFilesOnSameLevel && key.file != null && key.file.isDirectory()){
                	continue; // we only want those that are in the same directory, and not in other directories...
{'fraction_non_alphanumeric': 0.09498994331687695, 'fraction_numerical': 0.0009142439202779302, 'mean_word_length': 3.276387802971071, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6004653', 'n_tokens_mistral': 3030, 'n_tokens_neox': 2981, 'n_words': 789}