<commit_before>"""
@author <NAME>
@file pyon/util/file_sys.py
@description Utility for managing relative file system paths
"""

import errno
import StringIO
import tempfile
import shutil
import os
import re
import random
import string
from pyon.util.log import log
from pyon.util.containers import DotDict
from pyon.core.bootstrap import CFG as bootcfg


class FileSystemError(Exception):
    '''
    Client filesystem request failed
    Does this seem wrong to anyone else?!
    '''
    status_code = 411
    def get_status_code(self):
        return self.status_code

    def get_error_message(self):
        return self.message

    def __str__(self):
        return str(self.get_status_code()) + " - " + str(self.get_error_message())


class FileSystem(object):
    # These are static, and shared throughout the container, do not operate on a per-instance basis.
    FS_DIRECTORY_LIST = ['RESOURCE','TEMP','LIBRARY','CACHE','RUN','USERS','LOG']
    FS_DIRECTORY = DotDict(zip(FS_DIRECTORY_LIST,FS_DIRECTORY_LIST))

    FS = DotDict(zip(FS_DIRECTORY_LIST, FS_DIRECTORY_LIST))
    _instance = None


    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(FileSystem, cls).__new__(cls, *args, **kwargs)
        return cls._instance
    
    @classmethod 
    def _clean(cls, config):
        for k,v in FileSystem.FS_DIRECTORY.iteritems():
            s = v.lower() # Lower case string
            conf = config.get_safe('container.filesystem.%s' % s, None)
            if conf:
                FileSystem.FS_DIRECTORY[k] = conf
            else:
                FileSystem.FS_DIRECTORY[k] = os.path.join('/tmp',s)
            if os.path.exists(FS_DIRECTORY[k]):
                log.info('Removing %s' , FS_DIRECTORY[k])
                shutil.rmtree(FS_DIRECTORY[k])
                cls.__makedirs(FileSystem.FS_DIRECTORY[k]) # Catches race condition where concurrent container does the same thing


    def __init__(self, CFG):
        FileSystem._force_clean = CFG.get_safe('container.filesystem.force_clean',False)
        if FileSystem._force_clean:
            self._clean(CFG)

        for k,v in FileSystem.FS_DIRECTORY.iteritems():
            s = v.lower() # Lower case string
            conf = CFG.get_safe('container.filesystem.%s' % s, None)
            if conf:
                FileSystem.FS_DIRECTORY[k] = conf
            else:
                FileSystem.FS_DIRECTORY[k] = os.path.join('/tmp',s)
            # Check to make sure you're within your rights to access this
            if not FileSystem._sandbox(FileSystem.FS_DIRECTORY[k]):
                raise OSError('You are attempting to perform an operation beyond the scope of your permission. (%s is set to \'%s\')' % (k,FileSystem.FS_DIRECTORY[k]))
            if not os.path.exists(FS_DIRECTORY[k]):
                log.info('Making path: %s', FS_DIRECTORY[k])<commit_msg>Stops creation of folders during cleaning only operations.<commit_after>"""
@author <NAME>
@file pyon/util/file_sys.py
@description Utility for managing relative file system paths
"""

import errno
import StringIO
import tempfile
import shutil
import os
import re
import random
import string
from pyon.util.log import log
from pyon.util.containers import DotDict
from pyon.core.bootstrap import CFG as bootcfg


class FileSystemError(Exception):
    '''
    Client filesystem request failed
    Does this seem wrong to anyone else?!
    '''
    status_code = 411
    def get_status_code(self):
        return self.status_code

    def get_error_message(self):
        return self.message

    def __str__(self):
        return str(self.get_status_code()) + " - " + str(self.get_error_message())


class FileSystem(object):
    # These are static, and shared throughout the container, do not operate on a per-instance basis.
    FS_DIRECTORY_LIST = ['RESOURCE','TEMP','LIBRARY','CACHE','RUN','USERS','LOG']
    FS_DIRECTORY = DotDict(zip(FS_DIRECTORY_LIST,FS_DIRECTORY_LIST))

    FS = DotDict(zip(FS_DIRECTORY_LIST, FS_DIRECTORY_LIST))
    _instance = None


    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(FileSystem, cls).__new__(cls, *args, **kwargs)
        return cls._instance
    
    @classmethod 
    def _clean(cls, config):
        for k,v in FileSystem.FS_DIRECTORY.iteritems():
            s = v.lower() 
            conf = config.get_safe('container.filesystem.%s' % s, None)
            if conf:
                FileSystem.FS_DIRECTORY[k] = conf
            else:
                FileSystem.FS_DIRECTORY[k] = os.path.join('/tmp',s)
            if os.path.exists(FS_DIRECTORY[k]):
                log.info('Removing %s' , FS_DIRECTORY[k])
                shutil.rmtree(FS_DIRECTORY[k])


    def __init__(self, CFG):
        FileSystem._force_clean = CFG.get_safe('container.filesystem.force_clean',False)
        if FileSystem._force_clean:
            self._clean(CFG)

        for k,v in FileSystem.FS_DIRECTORY.iteritems():
            s = v.lower() # Lower case string
            conf = CFG.get_safe('container.filesystem.%s' % s, None)
            if conf:
                FileSystem.FS_DIRECTORY[k] = conf
            else:
                FileSystem.FS_DIRECTORY[k] = os.path.join('/tmp',s)
            # Check to make sure you're within your rights to access this
            if not FileSystem._sandbox(FileSystem.FS_DIRECTORY[k]):
                raise OSError('You are attempting to perform an operation beyond the scope of your permission. (%s is set to \'%s\')' % (k,FileSystem.FS_DIRECTORY[k]))
            if not os.path.exists(FS_DIRECTORY[k]):
                log.info('Making path: %s', FS_DIRECTORY[k])
{'fraction_non_alphanumeric': 0.10673465799191138, 'fraction_numerical': 0.0010550378055213644, 'mean_word_length': 2.6744186046511627, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5356909', 'n_tokens_mistral': 1761, 'n_tokens_neox': 1674, 'n_words': 434}