<commit_before>// PythonModuleTracker.java

/**
*    Copyright (C) 2008 10gen Inc.
*  
*    This program is free software: you can redistribute it and/or  modify
*    it under the terms of the GNU Affero General Public License, version 3,
*    as published by the Free Software Foundation.
*  
*    This program is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU Affero General Public License for more details.
*  
*    You should have received a copy of the GNU Affero General Public License
*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package ed.lang.python;

import org.python.core.*;
import java.util.*;
import java.io.*;

import ed.appserver.*;
import ed.util.*;

/**
 * Replacement class for sys.modules. Can flush outdated modules, and
 * prevents untrusted code from getting access to Java
 * packages/classes.
 */
public class PythonModuleTracker extends PyStringMap {
    static final boolean DEBUG = Boolean.getBoolean( "DEBUG.PYTHONMODULETRACKER" );

    boolean _flushing = false;

    PythonModuleTracker( PyStringMap old ){
        super();
        update( new PyObject[]{ old }, new String[0] );
    }

    public PyObject __finditem__( String key ){
        return handleReturn( super.__finditem__( key ) );
    }

    public PyObject __finditem__( PyObject key ){
        return handleReturn( super.__finditem__( key ) );
    }

    PyObject handleReturn( PyObject obj ){
        Python.checkSafeImport( obj );
        if( ! ( obj instanceof PyModule ) ) return obj;
        // if module is outdated
        PyModule module = (PyModule)obj;
        return module;
    }

    public PyObject get( PyObject key ){
        return get( key , Py.None );
    }

    public PyObject get( PyObject key , PyObject missing ){
        return handleReturn( super.get( key , missing ) );
    }

    public PyStringMap copy(){
        return new PythonModuleTracker( this );
    }

    public PyList keys(){
        PyList k = new PyList();
        PyList keys = super.keys();
        int len = keys.__len__();
        for( int i = 0; i < len; i++ ){
            PyObject key = keys.pyget( i );
            PyObject val = null;
            if( key instanceof PyString )
                val = super.__finditem__( key.toString() );
            else
                val = super.__finditem__( key );
            if( Python.isSafeImport( val ) )
                k.append(key);
        }
        return k;
    }

    public PyList items(){
        PyList l = new PyList();
        PyList keys = keys();
        int len = keys.__len__();
        for( int i = 0; i < len ; i++){
            PyObject key = keys.pyget(i);
            PyObject val = null;
            if( key instanceof PyString )
                val = super.__finditem__( key.toString() );
            else
                val = super.__finditem__( key );
            PyTuple t = new PyTuple(key, val);
            l.append(t);
        }
        return l;
    }

    public PyList values(){
        PyList v = new PyList();
        PyList values = super.values();
        int len = values.__len__();
        for( int i = 0; i < len; ++i){
            PyObject val = values.pyget(i);
            if( Python.isSafeImport( val ) )
                v.append(val);
        }
        return v;
    }

    // FIXME: clumsy -- actually implement?
    public PyObject iterkeys(){
        return keys().__iter__();
    }

    public PyObject itervalues(){
        return values().__iter__();
    }

    public PyObject iteritems(){
        return items().__iter__();
    }

    public PyObject pop( PyObject key ){
        return pop( key , Py.None );
    }

    public PyObject pop( PyObject key , PyObject value ){
        PyObject m = super.__finditem__( key );
        // Don't pop unless safe
        if( Python.isSafeImport( m ) )
            return handleReturn( super.pop( key , value ) );
        Python.checkSafeImport( m ); // raises exception
        return null;
    }

    public void __setitem__( String key , PyObject value ){
        super.__setitem__( key , value );
        handleSet( value );
    }

    public void __setitem__( PyObject key , PyObject value ){
        super.__setitem__( key , value );
        handleSet( value );
    }

    void handleSet( PyObject obj ){
        // Thought I could get the __file__ and check timestamp, but at this
        // point in time, the module doesn't have a __file__
    }

    public static boolean needsRefresh( String filename ){
        // Src for file
        File pyFile = new File( filename );
        String clsPath = filename.replace( ".py" , "$py.class" );
        // Compiled class file -- might not exist
        File clsFile = new File( clsPath );

        if( ! pyFile.exists() ){
            if( DEBUG )
                System.out.println("File was deleted: " + pyFile);
            return true;
        }

        if( clsFile.exists() &&
            pyFile.lastModified() > clsFile.lastModified() ){
            if( DEBUG )
                System.out.println("Newer " + pyFile + " " + clsFile);
            return true;
        }
        return false;
    }

    /**
     * @return the flushed files
     */
    public Set<File> flushOld(){

        boolean shouldFlush = false;
<commit_msg>Resolve relative paths in module __file__ against the root of the site.<commit_after>// PythonModuleTracker.java

/**
*    Copyright (C) 2008 10gen Inc.
*  
*    This program is free software: you can redistribute it and/or  modify
*    it under the terms of the GNU Affero General Public License, version 3,
*    as published by the Free Software Foundation.
*  
*    This program is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU Affero General Public License for more details.
*  
*    You should have received a copy of the GNU Affero General Public License
*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package ed.lang.python;

import org.python.core.*;
import java.util.*;
import java.io.*;

import ed.appserver.*;
import ed.util.*;

/**
 * Replacement class for sys.modules. Can flush outdated modules, and
 * prevents untrusted code from getting access to Java
 * packages/classes.
 */
public class PythonModuleTracker extends PyStringMap {
    static final boolean DEBUG = Boolean.getBoolean( "DEBUG.PYTHONMODULETRACKER" );

    boolean _flushing = false;

    PythonModuleTracker( PyStringMap old ){
        super();
        update( new PyObject[]{ old }, new String[0] );
    }

    public PyObject __finditem__( String key ){
        return handleReturn( super.__finditem__( key ) );
    }

    public PyObject __finditem__( PyObject key ){
        return handleReturn( super.__finditem__( key ) );
    }

    PyObject handleReturn( PyObject obj ){
        Python.checkSafeImport( obj );
        if( ! ( obj instanceof PyModule ) ) return obj;
        // if module is outdated
        PyModule module = (PyModule)obj;
        return module;
    }

    public PyObject get( PyObject key ){
        return get( key , Py.None );
    }

    public PyObject get( PyObject key , PyObject missing ){
        return handleReturn( super.get( key , missing ) );
    }

    public PyStringMap copy(){
        return new PythonModuleTracker( this );
    }

    public PyList keys(){
        PyList k = new PyList();
        PyList keys = super.keys();
        int len = keys.__len__();
        for( int i = 0; i < len; i++ ){
            PyObject key = keys.pyget( i );
            PyObject val = null;
            if( key instanceof PyString )
                val = super.__finditem__( key.toString() );
            else
                val = super.__finditem__( key );
            if( Python.isSafeImport( val ) )
                k.append(key);
        }
        return k;
    }

    public PyList items(){
        PyList l = new PyList();
        PyList keys = keys();
        int len = keys.__len__();
        for( int i = 0; i < len ; i++){
            PyObject key = keys.pyget(i);
            PyObject val = null;
            if( key instanceof PyString )
                val = super.__finditem__( key.toString() );
            else
                val = super.__finditem__( key );
            PyTuple t = new PyTuple(key, val);
            l.append(t);
        }
        return l;
    }

    public PyList values(){
        PyList v = new PyList();
        PyList values = super.values();
        int len = values.__len__();
        for( int i = 0; i < len; ++i){
            PyObject val = values.pyget(i);
            if( Python.isSafeImport( val ) )
                v.append(val);
        }
        return v;
    }

    // FIXME: clumsy -- actually implement?
    public PyObject iterkeys(){
        return keys().__iter__();
    }

    public PyObject itervalues(){
        return values().__iter__();
    }

    public PyObject iteritems(){
        return items().__iter__();
    }

    public PyObject pop( PyObject key ){
        return pop( key , Py.None );
    }

    public PyObject pop( PyObject key , PyObject value ){
        PyObject m = super.__finditem__( key );
        // Don't pop unless safe
        if( Python.isSafeImport( m ) )
            return handleReturn( super.pop( key , value ) );
        Python.checkSafeImport( m ); // raises exception
        return null;
    }

    public void __setitem__( String key , PyObject value ){
        super.__setitem__( key , value );
        handleSet( value );
    }

    public void __setitem__( PyObject key , PyObject value ){
        super.__setitem__( key , value );
        handleSet( value );
    }

    void handleSet( PyObject obj ){
        // Thought I could get the __file__ and check timestamp, but at this
        // point in time, the module doesn't have a __file__
    }

    public static boolean needsRefresh( String filename ){
        // Src for file
        File pyFile = new File( filename );
        if( ! pyFile.isAbsolute() )
            pyFile = new File( Py.getSystemState().getCurrentWorkingDir() , pyFile.toString() );
        String clsPath = filename.replace( ".py" , "$py.class" );
        // Compiled class file -- might not exist
        File clsFile = new File( clsPath );

        if( ! pyFile.exists() ){
            if( DEBUG )
                System.out.println("File was deleted: " + pyFile);
            return true;
        }

        if( clsFile.exists() &&
            pyFile.lastModified() > clsFile.lastModified() ){
            if( DEBUG )
                System.out.println("Newer " + pyFile + " " + clsFile);
            return true;
        }
        return false;
    }

    /**
     * @return the flushed files
     */
    public Set<File> flushOld(){

        boolean shouldFlush = false;

{'fraction_non_alphanumeric': 0.10284623079021551, 'fraction_numerical': 0.002000545603346367, 'mean_word_length': 1.9461559067773908, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2267351', 'n_tokens_mistral': 2992, 'n_tokens_neox': 2970, 'n_words': 1030}