<commit_before>import os
from uliweb import settings, request, url_for
from uliweb.utils import files
from uliweb.utils.common import import_attr, application_path, log
import random
import time

__all__ = ['save_file', 'get_filename', 'get_url', 'save_file_field', 'save_image_field', 
    'delete_filename', 'norm_filename']

default_fileserving = None

def norm_filename(filename):
    return os.path.normpath(filename).replace('\\', '/')

class FilenameConverter(object):
    @staticmethod
    def convert(filename):
        return filename
    
class UUIDFilenameConverter(object):
    @staticmethod
    def convert(filename):
        import uuid
        _f, ext = os.path.splitext(filename)
        return uuid.uuid1().hex + ext
    
class MD5FilenameConverter(object):
    @staticmethod
    def convert(filename):
        try:
            from hashlib import md5
        except ImportError:
            from md5 import md5

        _f, ext = os.path.splitext(filename)
        f = md5(
                    md5("%f%s%f%s" % (time.time(), id({}), random.random(),
                                      os.getpid())).hexdigest(), 
                ).hexdigest()
        
        return f + ext
    
class FileServing(object):
    default_config = 'UPLOAD'
    options = {
        'x_sendfile' : ('X_SENDFILE', None),
        'x_header_name': ('X_HEADER_NAME', ''),
        'x_file_prefix': ('X_FILE_PREFIX', '/files'),
        'to_path': ('TO_PATH', './uploads'),
        'buffer_size': ('BUFFER_SIZE', 4096),
        '_filename_converter': ('FILENAME_CONVERTER', None),
    }
    
    def __init__(self, default_filename_converter_cls=UUIDFilenameConverter, config=None):
        self.config = config or self.default_config
        for k, v in self.options.items():
            item, default = v
            #if there is no '/' in option, then combine config with option
            #else assume the option is just like 'SECTION/OPTION', then skip it
            if '/' not in item:
                item = self.config + '/' + item
            value = settings.get_var(item, default)
            setattr(self, k, value)
            
        if self.x_sendfile and not self.x_header_name:
            if self.x_sendfile == 'nginx':
                self.x_header_name = 'X-Accel-Redirect'
            elif self.x_sendfile == 'apache':
                self.x_header_name = 'X-Sendfile'
            else:
                raise Exception, "X_HEADER can't be None, or X_SENDFILE is not supprted"
        if isinstance(self._filename_converter, (str, unicode)):
            self._filename_converter_cls = import_attr(self._filename_converter)
        else:
            self._filename_converter_cls = self._filename_converter or default_filename_converter_cls
        
    def filename_convert(self, filename, convert_cls=None):
        convert_cls = convert_cls or self._filename_converter_cls
        return convert_cls.convert(filename)
        
    def get_filename(self, filename, filesystem=False, convert=False):
        """
        Get the filename according to self.to_path, and if filesystem is False
        then return unicode filename, otherwise return filesystem encoded filename
    
        @param filename: relative filename, it'll be combine with self.to_path
        @param filesystem: if True, then encoding the filename to filesystem
        @param convert: if True, then convert filename with FilenameConverter class
        """
        from uliweb.utils.common import safe_unicode
        
        #make sure the filename is unicode
        s = settings.GLOBAL
        if convert:
            _p, _f = os.path.split(filename)
            _filename = os.path.join(_p, self.filename_convert(_f))
        else:
            _filename = filename
        nfile = safe_unicode(_filename, s.HTMLPAGE_ENCODING)
        
        f = os.path.normpath(os.path.join(application_path(self.to_path), nfile)).replace('\\', '/')
    
        if filesystem:
            return files.encode_filename(f, to_encoding=s.FILESYSTEM_ENCODING)
        return f

    def download(self, filename, action='download', x_filename='', real_filename=''):
        """
        action will be "download", "inline"
        and if the request.GET has 'action', then the action will be replaced by it.
        """
        from uliweb.utils.common import safe_str
        from uliweb.utils.filedown import filedown
        
        s = settings.GLOBAL

        action = request.GET.get('action', action)
        
        if not real_filename:
            real_filename = self.get_filename(filename, True, convert=False)
        else:
            real_filename = files.encode_filename(real_filename, to_encoding=s.FILESYSTEM_ENCODING)

        if not x_filename:
            x_filename = safe_str(filename, s.FILESYSTEM_ENCODING)
        if self.x_file_prefix:
            x_filename = os.path.normpath(os.path.join(self.x_file_prefix, x_filename)).replace('\\', '/')
        
        return filedown(request.environ, filename, action=action, 
            x_sendfile=bool(self.x_sendfile), x_header_name=self.x_header_name, 
            x_filename=x_filename, real_filename=real_filename)
     
    def save_file(self, filename, fobj, replace=False, convert=True):
        from uliweb.utils import files
        
        #get full path and converted filename
        fname = self.get_filename(filename, True, convert=convert)
        #save file and get the changed filename, because the filename maybe change when
        #there is duplicate filename, if replace=True, then the filename
        #will not changed
        fname2 = files.save_file(fname, fobj, replace, self.buffer_size)
        
        s = settings.GLOBAL
        #create new filename according fname2 and filename, the result should be unicode
        return norm_filename(os.path.join(os.path.dirname(filename), files.unicode_filename(fname2, s.FILESYSTEM_ENCODING)))
    
    def save_file_field(self, field, replace=False, filename=None, convert=True):
        filename = filename or field.data.filename
        fname = self.save_file(filename, field.data.file, replace, convert)
        field.data.filename = fname
        return fname
            
    def save_image_field(self, field, resize_to=None, replace=False, filename=None, convert=True):
        from uliweb.utils.image import resize_image
        if resize_to:
            field.data.file = resize_image(field.data.file, resize_to)
        filename = filename or field.data.filename
        fname = self.save_file(filename, field.data.file, replace, convert)
        field.data.filename = fname
        return fname
            
    def delete_filename(self, filename):
        f = self.get_filename(filename, filesystem=True, convert=False)
        if os.path.exists(f):
            try:
                os.unlink(f)
            except Exception, e:
                log.exception(e)
    
    def get_href(self, filename, **kwargs):
        if not filename:
            return ''

        s = settings.GLOBAL
        fname = norm_filename(files.unicode_filename(filename, s.FILESYSTEM_ENCODING))
        f = url_for('file_serving', filename=fname, **kwargs)
        return f
        
    def get_url(self, filename, query_para=None, **url_args):
        """
        Return <a href="filename" title="filename"> tag
        You should pass title and text to url_args, if not pass, then using filename
        """
        from uliweb.core.html import Tag
        
        title = url_args.pop('title', filename)
        text = url_args.pop('text', title)
        query_para = query_para or {}
        return str(Tag('a', title, href=self.get_href(filename, **query_para), **url_args))

def get_backend(config=None):
    global default_fileserving
    
    if default_fileserving and not config:
        return default_fileserving
    else:
        config = config or 'UPLOAD'
        cls = settings.get_var('%s/BACKEND' % config)
        if cls:
            fileserving = import_attr(cls)(config=config)
        else:
            fileserving = FileServing(config=config)
        if config == 'UPLOAD':
            default_fileserving = fileserving
        return fileserving

get_fileserving = get_backend

def file_serving(filename):
    from uliweb import request
    import urllib2
    
    alt_filename = request.GET.get('alt')
    if not alt_filename:
        alt_filename = filename
    else:
        alt_filename = urllib2.unquote(alt_filename)
    _filename = get_filename(filename, False, convert=False)
    x_filename = filename
    return get_backend().download(alt_filename, real_filename=_filename, x_filename=x_filename, action=None)
<commit_msg>Fix file_serving add real_filename and x_filename parameter<commit_after>import os
from uliweb import settings, request, url_for
from uliweb.utils import files
from uliweb.utils.common import import_attr, application_path, log
import random
import time

__all__ = ['save_file', 'get_filename', 'get_url', 'save_file_field', 'save_image_field', 
    'delete_filename', 'norm_filename']

default_fileserving = None

def norm_filename(filename):
    return os.path.normpath(filename).replace('\\', '/')

class FilenameConverter(object):
    @staticmethod
    def convert(filename):
        return filename
    
class UUIDFilenameConverter(object):
    @staticmethod
    def convert(filename):
        import uuid
        _f, ext = os.path.splitext(filename)
        return uuid.uuid1().hex + ext
    
class MD5FilenameConverter(object):
    @staticmethod
    def convert(filename):
        try:
            from hashlib import md5
        except ImportError:
            from md5 import md5

        _f, ext = os.path.splitext(filename)
        f = md5(
                    md5("%f%s%f%s" % (time.time(), id({}), random.random(),
                                      os.getpid())).hexdigest(), 
                ).hexdigest()
        
        return f + ext
    
class FileServing(object):
    default_config = 'UPLOAD'
    options = {
        'x_sendfile' : ('X_SENDFILE', None),
        'x_header_name': ('X_HEADER_NAME', ''),
        'x_file_prefix': ('X_FILE_PREFIX', '/files'),
        'to_path': ('TO_PATH', './uploads'),
        'buffer_size': ('BUFFER_SIZE', 4096),
        '_filename_converter': ('FILENAME_CONVERTER', None),
    }
    
    def __init__(self, default_filename_converter_cls=UUIDFilenameConverter, config=None):
        self.config = config or self.default_config
        for k, v in self.options.items():
            item, default = v
            #if there is no '/' in option, then combine config with option
            #else assume the option is just like 'SECTION/OPTION', then skip it
            if '/' not in item:
                item = self.config + '/' + item
            value = settings.get_var(item, default)
            setattr(self, k, value)
            
        if self.x_sendfile and not self.x_header_name:
            if self.x_sendfile == 'nginx':
                self.x_header_name = 'X-Accel-Redirect'
            elif self.x_sendfile == 'apache':
                self.x_header_name = 'X-Sendfile'
            else:
                raise Exception, "X_HEADER can't be None, or X_SENDFILE is not supprted"
        if isinstance(self._filename_converter, (str, unicode)):
            self._filename_converter_cls = import_attr(self._filename_converter)
        else:
            self._filename_converter_cls = self._filename_converter or default_filename_converter_cls
        
    def filename_convert(self, filename, convert_cls=None):
        convert_cls = convert_cls or self._filename_converter_cls
        return convert_cls.convert(filename)
        
    def get_filename(self, filename, filesystem=False, convert=False):
        """
        Get the filename according to self.to_path, and if filesystem is False
        then return unicode filename, otherwise return filesystem encoded filename
    
        @param filename: relative filename, it'll be combine with self.to_path
        @param filesystem: if True, then encoding the filename to filesystem
        @param convert: if True, then convert filename with FilenameConverter class
        """
        from uliweb.utils.common import safe_unicode
        
        #make sure the filename is unicode
        s = settings.GLOBAL
        if convert:
            _p, _f = os.path.split(filename)
            _filename = os.path.join(_p, self.filename_convert(_f))
        else:
            _filename = filename
        nfile = safe_unicode(_filename, s.HTMLPAGE_ENCODING)
        
        f = os.path.normpath(os.path.join(application_path(self.to_path), nfile)).replace('\\', '/')
    
        if filesystem:
            return files.encode_filename(f, to_encoding=s.FILESYSTEM_ENCODING)
        return f

    def download(self, filename, action='download', x_filename='', real_filename=''):
        """
        action will be "download", "inline"
        and if the request.GET has 'action', then the action will be replaced by it.
        """
        from uliweb.utils.common import safe_str
        from uliweb.utils.filedown import filedown
        
        s = settings.GLOBAL

        action = request.GET.get('action', action)
        
        if not real_filename:
            real_filename = self.get_filename(filename, True, convert=False)
        else:
            real_filename = files.encode_filename(real_filename, to_encoding=s.FILESYSTEM_ENCODING)

        if not x_filename:
            x_filename = safe_str(filename, s.FILESYSTEM_ENCODING)
        if self.x_file_prefix:
            x_filename = os.path.normpath(os.path.join(self.x_file_prefix, x_filename)).replace('\\', '/')
        
        return filedown(request.environ, filename, action=action, 
            x_sendfile=bool(self.x_sendfile), x_header_name=self.x_header_name, 
            x_filename=x_filename, real_filename=real_filename)
     
    def save_file(self, filename, fobj, replace=False, convert=True):
        from uliweb.utils import files
        
        #get full path and converted filename
        fname = self.get_filename(filename, True, convert=convert)
        #save file and get the changed filename, because the filename maybe change when
        #there is duplicate filename, if replace=True, then the filename
        #will not changed
        fname2 = files.save_file(fname, fobj, replace, self.buffer_size)
        
        s = settings.GLOBAL
        #create new filename according fname2 and filename, the result should be unicode
        return norm_filename(os.path.join(os.path.dirname(filename), files.unicode_filename(fname2, s.FILESYSTEM_ENCODING)))
    
    def save_file_field(self, field, replace=False, filename=None, convert=True):
        filename = filename or field.data.filename
        fname = self.save_file(filename, field.data.file, replace, convert)
        field.data.filename = fname
        return fname
            
    def save_image_field(self, field, resize_to=None, replace=False, filename=None, convert=True):
        from uliweb.utils.image import resize_image
        if resize_to:
            field.data.file = resize_image(field.data.file, resize_to)
        filename = filename or field.data.filename
        fname = self.save_file(filename, field.data.file, replace, convert)
        field.data.filename = fname
        return fname
            
    def delete_filename(self, filename):
        f = self.get_filename(filename, filesystem=True, convert=False)
        if os.path.exists(f):
            try:
                os.unlink(f)
            except Exception, e:
                log.exception(e)
    
    def get_href(self, filename, **kwargs):
        if not filename:
            return ''

        s = settings.GLOBAL
        fname = norm_filename(files.unicode_filename(filename, s.FILESYSTEM_ENCODING))
        f = url_for('file_serving', filename=fname, **kwargs)
        return f
        
    def get_url(self, filename, query_para=None, **url_args):
        """
        Return <a href="filename" title="filename"> tag
        You should pass title and text to url_args, if not pass, then using filename
        """
        from uliweb.core.html import Tag
        
        title = url_args.pop('title', filename)
        text = url_args.pop('text', title)
        query_para = query_para or {}
        return str(Tag('a', title, href=self.get_href(filename, **query_para), **url_args))

def get_backend(config=None):
    global default_fileserving
    
    if default_fileserving and not config:
        return default_fileserving
    else:
        config = config or 'UPLOAD'
        cls = settings.get_var('%s/BACKEND' % config)
        if cls:
            fileserving = import_attr(cls)(config=config)
        else:
            fileserving = FileServing(config=config)
        if config == 'UPLOAD':
            default_fileserving = fileserving
        return fileserving

get_fileserving = get_backend

def file_serving(filename, real_filename=None, x_filename=None):
    from uliweb import request
    import urllib2
    
    alt_filename = request.GET.get('alt')
    if not alt_filename:
        alt_filename = filename
    else:
        alt_filename = urllib2.unquote(alt_filename)
    if not real_filename:
        _filename = get_filename(filename, False, convert=False)
    else:
        _filename = real_filename
    if not x_filename:
        x_filename = filename
    return get_backend().download(alt_filename, real_filename=_filename, x_filename=x_filename, action=None)

{'fraction_non_alphanumeric': 0.0932648401826484, 'fraction_numerical': 0.0018264840182648401, 'mean_word_length': 2.475699266018647, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1537088', 'n_tokens_mistral': 5036, 'n_tokens_neox': 4936, 'n_words': 1261}