<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# =============================================================================
## @file ostap/utils/cleanup.py
#  Module with some simple but useful utilities
#  to deal  with temporary files and directories
#
#  @author <NAME> <EMAIL>
#  @date   2013-02-10  
# =============================================================================
"""Module with some simple but useful utilities
to deal  with temporary files and directories 
Module with some simple but useful utilities for
"""
# =============================================================================
__version__ = "$Revision$"
__author__  = "<NAME> <EMAIL>"
__date__    = "2013-02-10"
__all__     = (
    'CleanUp'  , ## Simple (base) class to deal with temporary files and directories
    'TempFile' , ## Simple placeholder for temporary file  
    )
# =============================================================================
import os, tempfile, datetime  
from   sys import version_info as python_version 
# =============================================================================
from   ostap.logger.logger import getLogger
if '__main__' ==  __name__ : logger = getLogger( 'ostap.utils.cleanup' )
else                       : logger = getLogger( __name__              )
del getLogger
from ostap.core.ostap_types import string_types
# =============================================================================
## temporary directory for <code>tempfile</code> module
_TmpDir = None
if not _TmpDir :
    ## 1) check the environment variable OSTAP_TMPDIR 
    _TmpDir = os.environ.get  ( 'OSTAP_TMPDIR' , None )
if not _TmpDir :
    ## 2) check the configuration file 
    import ostap.core.config as OCC 
    _TmpDir = OCC.general.get ( 'TmpDir' , None )
    del OCC 
if not _TmpDir : _TmpDir = None
# ===============================================================================
## Context manager to define/redefine TmpDir for <code>tempfile</code> module
class UseTmpDir ( object ) :
    """Context manager to define/redefine TmpDir for the tempfile module
    """
    def __init__   ( self , tmp_dir = None ) :
        self.tmp_dir  = tmp_dir
        self.previous = None 
        
    def __enter__  ( self ) :
        self.previous    = tempfile.tempdir
        tempfile.tempdir = self.tmp_dir
        
    def __exit__   ( self , *_ ) :
        if self.previous :
            tempfile.tempdir = self.previous 


## # =============================================================================
## ## clean an ancient stuff from TMP directory
## def clean_ancient_stuff ( what = _TmpDir , startwith = '/tmp' ) :
    
##     with UseTmpDir ( what ) :
##         tdir = tempfile.gettempdir()
##         if os.path.exist ( tdir ) and os.path.isdir ( tdir ) :
##             import getpass
##             username = getpass.getuser()
##             if tdir.startswith ( startdir ) :
##                 commandp = 'find %s -type f -atime +1 -print' % tdir 
##                 commandd = 'find %s -type f -atime +1 -print' % tdir 
##                 import subprocess
##                 pp = bprocess.Pipe ( commandp.split() ,
##                                      stdout = subprocess.PIPE ,
##                                      stderr = subprocess.PIPE )
                
##                 op , ep = pp.communucate ()
##                 op = op.split ( '\n' )
##                 ep = ep.split ( '\n' )
##                 if '' in op : op.remove  ('')
##                 if '' in ep : ep.remove  ('')
##                 ppc = pp.returncode
##                 if ppc or ep : pass
                
##                 pd = bprocess.Pipe ( commandd.split() ,
##                                      stdout = subprocess.PIPE ,
##                                      stderr = subprocess.PIPE )
                                
##                 od , ed = pd.communucate ()
##                 od = od.split ( '\n' )
##                 ed = ed.split ( '\n' )
##                 if '' in od : od.remove  ('')
##                 if '' in ed : ed.remove  ('')
##                 pdc = pd.returncode
##                 if pdc or ed : pass

# =============================================================================
## @class CleanUp
#  Simple (base) class to get temporary files
#  and directories and to remove them at exit
class  CleanUp(object) :
    """Simple (base) class to get temporary files
    and directories and to remove them at exit
    """
    _tmpfiles  = set ()
    _tmpdirs   = set ()
    _protected = set ()
    
    ## @attention ensure that important attributes are available even before __init__
    def __new__( cls, *args, **kwargs):
        if  python_version.major > 2 : obj = super(CleanUp, cls).__new__( cls )
        else                         : obj = super(CleanUp, cls).__new__( cls , *args , **kwargs )
        ## define the local trash 
        obj.__trash = set() 
        return obj
 
    @property
    def tmpdir ( self ) :
        """``tmpdir'' : return the name of temporary managed directory
        - the managed directory will be cleaned-up and deleted at-exit
        >>> o    = CleanUp()
        >>> tdir = o.tmpdir 
        """
        tdir = CleanUp.tempdir () 
        return tdir
    
    @property
    def tmpdirs  ( self ) :
        """``tmpdirs'' - list of currently registered managed temporary directories"""
        return tuple ( self._tmpdirs )
    
    @tmpdirs.setter
    def tmpdirs ( self, values ) :
        if instance ( values , str ) : values = [ values ]
        for o in values :
            if o and isinstance ( o ,  str ) :
                self._tmpdirs.add ( o )
                logger.verbose ( 'temporary directory     added %s' % o )
                
    @property 
    def tmpfiles ( self ) :
        """``tempfiles'' : list of registered managed temporary files"""
        return list ( self._tmpfiles )
    
    @tmpfiles.setter
    def tmpfiles ( self , other ) :
        if isinstance ( other , str ) : other = [ other ]
        for o in other :
            if o and isinstance ( o , str ) : 
                self._tmpfiles.add ( o )
                logger.verbose ( 'temporary file          added %s' % o )

    @property
    def trash ( self ) :
        """``trash'' : local trash (files, directory), to be removed at deletion of the instance"""
        return self.__trash
    
    ## delete all local trash (files, directory)
    def __del__ ( self ) :
        """Delete all local trash (files, directory)
        """
        while self.__trash : self.remove ( self.__trash.pop () )
            
    @staticmethod
    def tempdir ( suffix = '' , prefix = 'tmp-' , date = True ) :
        """Get the name of the temporary directory.
        The directory will be cleaned-up and deleted at-exit.
        >>> dirname = CleanUp.tempdir() 
        """
        with UseTmpDir ( _TmpDir ) :
            if date :
                now = datetime.datetime.now()
                prefix = "%s%s-"   %  ( prefix , now.strftime ( "%Y-%b-%d" ) )
            tmp = tempfile.mkdtemp ( suffix = suffix , prefix = prefix ) 
            CleanUp._tmpdirs.add ( tmp )
            logger.verbose ( 'temporary directory requested %s' % tmp   )
            return tmp        

    
    @staticmethod
    def get_temp_file ( suffix = '' , prefix = 'tmp-' , dir = None , date = True ) :
        """Generate the name for the temporary file.
        - the method shodul be  abvoided in favour of` CleanUp.tempfile`
        >>> fname = CleanUp.get_temp_file () 
        """
        with UseTmpDir ( _TmpDir ) :
            if date :
                now = datetime.datetime.now()
                prefix = "%s%s-"   %  ( prefix , now.strftime ( "%Y-%b-%d" ) )            
            _file = tempfile.NamedTemporaryFile ( suffix = suffix ,
                                                  prefix = prefix ,
                                                  dir    = dir    , 
                                                  delete = False  )
            fname = _file.name
            _file.close()
            os.unlink(fname)
            assert not os.path.exists ( fname )
            logger.verbose  ( 'temporary file      requested %s' % fname )
            return fname

    @staticmethod
    def tempfile ( suffix = '' , prefix = 'tmp-' , dir = None , date = True ) :
        """Get the name of the temporary file.
        - The file will be deleted at-exit
        >>> fname = CleanUp.tempfile() 
        """
        fname = CleanUp.get_temp_file ( suffix = suffix , prefix = prefix ,
                                        dir    = dir    , date   = date   )
        assert not os.path.exists  ( fname )
        CleanUp._tmpfiles.add ( fname )
        return fname

    @staticmethod
    def protect_file ( fname ) :
        """Protect the temporary from removal"""        
        if os.path.exists ( fname ) and os.path.isfile ( fname ) :
            CleanUp._protected.add ( fname ) 
            logger.verbose ( 'the file is protected: %s ' % fname )
            
    @staticmethod
    def remove_file ( fname ) :
        """Remove the (temporary) file
        """<commit_msg> tiny fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# =============================================================================
## @file ostap/utils/cleanup.py
#  Module with some simple but useful utilities
#  to deal  with temporary files and directories
#
#  @author <NAME> <EMAIL>
#  @date   2013-02-10  
# =============================================================================
"""Module with some simple but useful utilities
to deal  with temporary files and directories 
Module with some simple but useful utilities for
"""
# =============================================================================
__version__ = "$Revision$"
__author__  = "<NAME> <EMAIL>"
__date__    = "2013-02-10"
__all__     = (
    'CleanUp'  , ## Simple (base) class to deal with temporary files and directories
    'TempFile' , ## Simple placeholder for temporary file  
    )
# =============================================================================
import os, tempfile, datetime  
from   sys import version_info as python_version 
# =============================================================================
from   ostap.logger.logger import getLogger
if '__main__' ==  __name__ : logger = getLogger( 'ostap.utils.cleanup' )
else                       : logger = getLogger( __name__              )
del getLogger
from ostap.core.ostap_types import string_types
# =============================================================================
## temporary directory for <code>tempfile</code> module
_TmpDir = None
if not _TmpDir :
    ## 1) check the environment variable OSTAP_TMPDIR 
    _TmpDir = os.environ.get  ( 'OSTAP_TMPDIR' , None )
if not _TmpDir :
    ## 2) check the configuration file 
    import ostap.core.config as OCC 
    _TmpDir = OCC.general.get ( 'TmpDir' , None )
    del OCC 
if not _TmpDir : _TmpDir = None
# ===============================================================================
## Context manager to define/redefine TmpDir for <code>tempfile</code> module
class UseTmpDir ( object ) :
    """Context manager to define/redefine TmpDir for the tempfile module
    """
    def __init__   ( self , tmp_dir = None ) :
        self.tmp_dir  = tmp_dir
        self.previous = None 
        
    def __enter__  ( self ) :
        self.previous    = tempfile.tempdir
        tempfile.tempdir = self.tmp_dir
        
    def __exit__   ( self , *_ ) :
        if self.previous :
            tempfile.tempdir = self.previous 


## # =============================================================================
## ## clean an ancient stuff from TMP directory
## def clean_ancient_stuff ( what = _TmpDir , startwith = '/tmp' ) :
    
##     with UseTmpDir ( what ) :
##         tdir = tempfile.gettempdir()
##         if os.path.exist ( tdir ) and os.path.isdir ( tdir ) :
##             import getpass
##             username = getpass.getuser()
##             if tdir.startswith ( startdir ) :
##                 commandp = 'find %s -type f -atime +1 -print' % tdir 
##                 commandd = 'find %s -type f -atime +1 -print' % tdir 
##                 import subprocess
##                 pp = bprocess.Pipe ( commandp.split() ,
##                                      stdout = subprocess.PIPE ,
##                                      stderr = subprocess.PIPE )
                
##                 op , ep = pp.communucate ()
##                 op = op.split ( '\n' )
##                 ep = ep.split ( '\n' )
##                 if '' in op : op.remove  ('')
##                 if '' in ep : ep.remove  ('')
##                 ppc = pp.returncode
##                 if ppc or ep : pass
                
##                 pd = bprocess.Pipe ( commandd.split() ,
##                                      stdout = subprocess.PIPE ,
##                                      stderr = subprocess.PIPE )
                                
##                 od , ed = pd.communucate ()
##                 od = od.split ( '\n' )
##                 ed = ed.split ( '\n' )
##                 if '' in od : od.remove  ('')
##                 if '' in ed : ed.remove  ('')
##                 pdc = pd.returncode
##                 if pdc or ed : pass

# =============================================================================
## @class CleanUp
#  Simple (base) class to get temporary files
#  and directories and to remove them at exit
class  CleanUp(object) :
    """Simple (base) class to get temporary files
    and directories and to remove them at exit
    """
    _tmpfiles  = set ()
    _tmpdirs   = set ()
    _protected = set ()
    
    ## @attention ensure that important attributes are available even before __init__
    def __new__( cls, *args, **kwargs):
        if  python_version.major > 2 : obj = super(CleanUp, cls).__new__( cls )
        else                         : obj = super(CleanUp, cls).__new__( cls , *args , **kwargs )
        ## define the local trash 
        obj.__trash = set() 
        return obj
 
    @property
    def tmpdir ( self ) :
        """``tmpdir'' : return the name of temporary managed directory
        - the managed directory will be cleaned-up and deleted at-exit
        >>> o    = CleanUp()
        >>> tdir = o.tmpdir 
        """
        tdir = CleanUp.tempdir () 
        return tdir
    
    @property
    def tmpdirs  ( self ) :
        """``tmpdirs'' - list of currently registered managed temporary directories"""
        return tuple ( self._tmpdirs )
    
    @tmpdirs.setter
    def tmpdirs ( self, values ) :
        if instance ( values , str ) : values = [ values ]
        for o in values :
            if o and isinstance ( o ,  str ) :
                self._tmpdirs.add ( o )
                logger.verbose ( 'temporary directory     added %s' % o )
                
    @property 
    def tmpfiles ( self ) :
        """``tempfiles'' : list of registered managed temporary files"""
        return list ( self._tmpfiles )
    
    @tmpfiles.setter
    def tmpfiles ( self , other ) :
        if isinstance ( other , str ) : other = [ other ]
        for o in other :
            if o and isinstance ( o , str ) : 
                self._tmpfiles.add ( o )
                logger.verbose ( 'temporary file          added %s' % o )

    @property
    def trash ( self ) :
        """``trash'' : local trash (files, directory), to be removed at deletion of the instance"""
        return self.__trash
    
    ## delete all local trash (files, directory)
    def __del__ ( self ) :
        """Delete all local trash (files, directory)
        """
        while self.__trash : self.remove ( self.__trash.pop () )
            
    @staticmethod
    def tempdir ( suffix = '' , prefix = 'tmp-' , date = True ) :
        """Get the name of the temporary directory.
        The directory will be cleaned-up and deleted at-exit.
        >>> dirname = CleanUp.tempdir() 
        """
        with UseTmpDir ( _TmpDir ) :
            if date :
                now = datetime.datetime.now()
                prefix = "%s%s-"   %  ( prefix , now.strftime ( "%Y-%b-%d" ) )
            tmp = tempfile.mkdtemp ( suffix = suffix , prefix = prefix ) 
            CleanUp._tmpdirs.add ( tmp )
            logger.verbose ( 'temporary directory requested %s' % tmp   )
            return tmp        

    
    @staticmethod
    def get_temp_file ( suffix = '' , prefix = 'tmp-' , dir = None , date = True ) :
        """Generate the name for the temporary file.
        - the method should be  avoided in favour of `CleanUp.tempfile`
        >>> fname = CleanUp.get_temp_file () 
        """
        with UseTmpDir ( _TmpDir ) :
            if date :
                now = datetime.datetime.now()
                prefix = "%s%s-"   %  ( prefix , now.strftime ( "%Y-%b-%d" ) )            

            with tempfile.NamedTemporaryFile ( suffix = suffix ,
                                               prefix = prefix ,
                                               dir    = dir    , 
                                               delete = False  ) as tfile :
                fname = tfile.name
                
            os.unlink ( fname )            
            assert not os.path.exists ( fname )
            logger.verbose  ( 'temporary file      requested %s' % fname )
            return fname

    @staticmethod
    def tempfile ( suffix = '' , prefix = 'tmp-' , dir = None , date = True ) :
        """Get the name of the temporary file.
        - The file will be deleted at-exit
        >>> fname = CleanUp.tempfile() 
        """
        fname = CleanUp.get_temp_file ( suffix = suffix , prefix = prefix ,
                                        dir    = dir    , date   = date   )
        assert not os.path.exists  ( fname )
        CleanUp._tmpfiles.add ( fname )
        return fname

    @staticmethod
    def protect_file ( fname ) :
        """Protect the temporary from removal"""        
        if os.path.exists ( fname ) and os.path.isfile ( fname ) :
            CleanUp._protected.add ( fname ) 
            logger.verbose ( 'the file is protected: %s ' % fname )
            
    @staticmethod
    def remove_file ( fname ) :
        """Remove the (temporary) file
        """
{'fraction_non_alphanumeric': 0.17450649845016042, 'fraction_numerical': 0.0023927347871009843, 'mean_word_length': 1.763335837716003, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5411531', 'n_tokens_mistral': 4970, 'n_tokens_neox': 4762, 'n_words': 1456}