<commit_before>/*
 * Created on Nov 12, 2004
 *
 * @author <NAME>
 */
package org.python.pydev.editor.codecompletion.revisited;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.Serializable;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.Path;
import org.eclipse.jface.text.IDocument;
import org.python.pydev.core.FullRepIterable;
import org.python.pydev.core.REF;
import org.python.pydev.plugin.PydevPlugin;

/**
 * This is not a singleton because we may have a different pythonpath for each project (even though
 * we have a default one as the original pythonpath).
 * 
 * @author <NAME>
 */
public class PythonPathHelper implements Serializable{
    
    private static final long serialVersionUID = 1L;

    /**
     * This is a list of Files containg the pythonpath.
     */
    public List<String> pythonpath = new ArrayList<String>();
    
    /**
     * Returns the default path given from the string.
     * @param str
     * @param acceptPoint says if we can have dots in the str that has the path to be analyzed
     * @return a trimmed string with all the '\' converted to '/'
     */
    public String getDefaultPathStr(String str, boolean acceptPoint){
        if(acceptPoint == false && str.indexOf(".") == 0){ //cannot start with a dot
            throw new RuntimeException("The pythonpath can only have absolute paths (cannot start with '.', therefore, the path: '"+str+"' is not valid.");
        }
        File file = new File(str);
        if(file.exists()){
            str = REF.getFileAbsolutePath(file);
        }
        return str.trim().replaceAll("\\\\","/");
    }
    
    public String getDefaultPathStr(String str){
        return getDefaultPathStr(str, false);
    }
    
    /**
     * This method returns all modules that can be obtained from a root File.
     * @param monitor
     * @return the files in position 0 and folders in position 1.
     */
    public List<File>[] getModulesBelow(File root, IProgressMonitor monitor){
        if(!root.exists()){
            return null;
        }
        
        if(root.isDirectory()){
            FileFilter filter = new FileFilter() {
                
    	        public boolean accept(File pathname) {
    	            if(pathname.isFile()){
    	                return isValidFileMod(REF.getFileAbsolutePath(pathname));
    	            }else if(pathname.isDirectory()){
    	                return isFileOrFolderWithInit(pathname);
    	            }else{
    	                return false;
    	            }
    	        }
    	
    	    };
    	    return PydevPlugin.getPyFilesBelow(root, filter, monitor, true);
            
        }
        return null;
    }
    
    /**
     * @param root the zip file to analyze
     * @param monitor the monitor, to keep track of what is happening
     * @return a list with the name of the found modules in the jar
     */
    public static List<String> getFromJar(File root, IProgressMonitor monitor){
         String fileName = root.getName();
         if(root.isFile() && (fileName.endsWith(".jar") || fileName.endsWith(".zip"))){ //ok, it may be a jar file, so let's get its contents and get the available modules
            Set<String> folders = new HashSet<String>();
            try {
                String zipFileName = root.getName();
                ZipFile zipFile = new ZipFile(root);
                Enumeration<? extends ZipEntry> entries = zipFile.entries();
                
                //ok, now that we have the zip entries, let's map them to modules
                while(entries.hasMoreElements()){
                    ZipEntry entry = entries.nextElement();
                    String name = entry.getName();
                    if(!entry.isDirectory()){
                        //it is a file... we will ignore them, as java files do not map to actual modules as python, but to classes.
                        //and will only add its parent folder...
                        IPath path = new Path(name);
                        String fileExtension = path.getFileExtension();
                        if(fileExtension != null && fileExtension.equals("class")){
                            path = path.removeFileExtension().removeLastSegments(1); //remove the class and the public class name
                            StringBuffer buffer = new StringBuffer();
                            for (int i = 0; i < path.segmentCount(); i++) {
                                buffer.append(path.segment(i));
                                String modName = buffer.toString();
                                
                                folders.add(modName);
                                if(i+1 < path.segmentCount()){
                                    buffer.append("."); //not the last one...
                                }
                            }
                            monitor.setTaskName("Found in "+zipFileName+" module "+buffer);
                        }
                    }
                }
                
                return new ArrayList<String>(folders);
            } catch (Exception e) {
                //that's ok, it is probably not a zip file after all...
                PydevPlugin.log(e);
            }
        }
        return null;
    }

    /**
     * 
     * @param path the path we want to analyze
     * @return if the path passed belongs to a valid python compiled extension
     */
    public static boolean isValidDll(String path){
        if( path.endsWith(".pyd")  ||
	        path.endsWith(".so")  ||
	        path.endsWith(".dll")){
            return true;
        }
        return false;
    }
    
    /**
     * @param path
     * @return if the path passed belongs to a valid python source file
     */
    public static boolean isValidSourceFile(String path) {
        path = path.toLowerCase();
        return path.endsWith(".py") || path.endsWith(".pyw");
    }
    
    public static boolean isValidSourceFile(IFile file) {
        String ext = file.getFileExtension().toLowerCase();
        return ext.equals("py") || ext.equals("pyw");
    }

    
    /**
     * 
     * @param path
     * @return if the paths maps to a valid python module (depending on its extension).
     */
    public static boolean isValidFileMod(String path){

        boolean ret = false;
        if( isValidSourceFile(path)){
             ret = true;
             
        } else if(isValidDll(path)){
            ret = true;
        }
        
        return ret;<commit_msg>*** empty log message ***<commit_after>/*
 * Created on Nov 12, 2004
 *
 * @author <NAME>
 */
package org.python.pydev.editor.codecompletion.revisited;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.Serializable;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.Path;
import org.eclipse.jface.text.IDocument;
import org.python.pydev.core.FullRepIterable;
import org.python.pydev.core.REF;
import org.python.pydev.plugin.PydevPlugin;

/**
 * This is not a singleton because we may have a different pythonpath for each project (even though
 * we have a default one as the original pythonpath).
 * 
 * @author <NAME>
 */
public class PythonPathHelper implements Serializable{
    
    private static final long serialVersionUID = 1L;

    /**
     * This is a list of Files containg the pythonpath.
     */
    public List<String> pythonpath = new ArrayList<String>();
    
    /**
     * Returns the default path given from the string.
     * @param str
     * @param acceptPoint says if we can have dots in the str that has the path to be analyzed
     * @return a trimmed string with all the '\' converted to '/'
     */
    public String getDefaultPathStr(String str, boolean acceptPoint){
        if(acceptPoint == false && str.indexOf(".") == 0){ //cannot start with a dot
            throw new RuntimeException("The pythonpath can only have absolute paths (cannot start with '.', therefore, the path: '"+str+"' is not valid.");
        }
        File file = new File(str);
        if(file.exists()){
            str = REF.getFileAbsolutePath(file);
        }
        return str.trim().replaceAll("\\\\","/");
    }
    
    public String getDefaultPathStr(String str){
        return getDefaultPathStr(str, false);
    }
    
    /**
     * This method returns all modules that can be obtained from a root File.
     * @param monitor
     * @return the files in position 0 and folders in position 1.
     */
    public List<File>[] getModulesBelow(File root, IProgressMonitor monitor){
        if(!root.exists()){
            return null;
        }
        
        if(root.isDirectory()){
            FileFilter filter = new FileFilter() {
                
    	        public boolean accept(File pathname) {
    	            if(pathname.isFile()){
    	                return isValidFileMod(REF.getFileAbsolutePath(pathname));
    	            }else if(pathname.isDirectory()){
    	                return isFileOrFolderWithInit(pathname);
    	            }else{
    	                return false;
    	            }
    	        }
    	
    	    };
    	    return PydevPlugin.getPyFilesBelow(root, filter, monitor, true);
            
        }
        return null;
    }
    
    /**
     * @param root the zip file to analyze
     * @param monitor the monitor, to keep track of what is happening
     * @return a list with the name of the found modules in the jar
     */
    public static List<String> getFromJar(File root, IProgressMonitor monitor){
         String fileName = root.getName();
         if(root.isFile() && (fileName.endsWith(".jar") || fileName.endsWith(".zip"))){ //ok, it may be a jar file, so let's get its contents and get the available modules
            Set<String> folders = new HashSet<String>();
            try {
                String zipFileName = root.getName();
                ZipFile zipFile = new ZipFile(root);
                Enumeration<? extends ZipEntry> entries = zipFile.entries();
                
                //ok, now that we have the zip entries, let's map them to modules
                while(entries.hasMoreElements()){
                    ZipEntry entry = entries.nextElement();
                    String name = entry.getName();
                    if(!entry.isDirectory()){
                        //it is a file... we will ignore them, as java files do not map to actual modules as python, but to classes.
                        //and will only add its parent folder...
                        IPath path = new Path(name);
                        String fileExtension = path.getFileExtension();
                        if(fileExtension != null && fileExtension.equals("class")){
                            path = path.removeFileExtension().removeLastSegments(1); //remove the class and the public class name
                            StringBuffer buffer = new StringBuffer();
                            for (int i = 0; i < path.segmentCount(); i++) {
                                buffer.append(path.segment(i));
                                String modName = buffer.toString();
                                
                                folders.add(modName);
                                if(i+1 < path.segmentCount()){
                                    buffer.append("."); //not the last one...
                                }
                            }
                            monitor.setTaskName("Found in "+zipFileName+" module "+buffer);
                        }
                    }
                }
                
                return new ArrayList<String>(folders);
            } catch (Exception e) {
                //that's ok, it is probably not a zip file after all...
                PydevPlugin.log(e);
            }
        }
        return null;
    }

    /**
     * 
     * @param path the path we want to analyze
     * @return if the path passed belongs to a valid python compiled extension
     */
    public static boolean isValidDll(String path){
        if( path.endsWith(".pyd")  ||
	        path.endsWith(".so")  ||
	        path.endsWith(".dll")){
            return true;
        }
        return false;
    }
    
    /**
     * @param path
     * @return if the path passed belongs to a valid python source file
     */
    public static boolean isValidSourceFile(String path) {
        path = path.toLowerCase();
        return path.endsWith(".py") || path.endsWith(".pyw");
    }
    
    public static boolean isValidSourceFile(IFile file) {
        String ext = file.getFileExtension();
        if(ext == null){ // no extension
        	return false;
        }
		ext = ext.toLowerCase();
        return ext.equals("py") || ext.equals("pyw");
    }

    
    /**
     * 
     * @param path
     * @return if the paths maps to a valid python module (depending on its extension).
     */
    public static boolean isValidFileMod(String path){

        boolean ret = false;
        if( isValidSourceFile(path)){
             ret = true;
             
        } else if(isValidDll(path)){
            ret = true;
        }
        
        return ret;
{'fraction_non_alphanumeric': 0.1010634551728995, 'fraction_numerical': 0.001831114867244172, 'mean_word_length': 1.7680311890838207, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6821110', 'n_tokens_mistral': 3750, 'n_tokens_neox': 3625, 'n_words': 1137}