<commit_before>"""
    :copyright: (c) 2011 Local Projects, all rights reserved
    :license: Affero GNU GPL v3, see LICENSE for more details.
"""

import cStringIO
import traceback
import framework.util as util
import os
from framework.s3uploader import S3Uploader
from framework.log import log
from framework.controller import Controller
from framework.config import Config
from PIL import Image, ImageOps

class FileServer(object):
    """
    A generic FileServer.
    
    """
    
    def add(self, data, filename=None, make_unique=False, max_size=None, grayscale=False, mirror=True, thumb_max_size=None):
        """
        Add a file to the fileserver.  If either adding the database record for 
        the file or saving the file fails, then add will return None, and no
        modification will be made.  Otherwise, the ID of the record in the 
        database will be returned.
        
        """
        log.info("FileServer.add")
        
        # Create a unique filename
        if not filename or make_unique:
            filename = self.getUniqueFilename(filename)
        
        # Save the file to the system
        success = self.saveFile(filename, data, max_size=max_size, mirror=mirror)
        
        # Return the media_id of the file
        return filename
    
    
    def getUniqueFilename(self, filename):
        """
        Get a unique name for the file. 
        
        """
        raise NotImplementedError
    
    
    def addDbRecord(self, db, filename):
        """
        Insert a new record for a file into the given database.
        
        Arguments:
        db -- A web.py database (`web.db`) object
        app -- The name of the app (`str`)
        
        """
        try:
            id = db.insert('attachments', title=filename)
        except Exception, e:
            log.error(e)
            return None
        
        return id
    
    
    def removeDbRecord(self, db, id):
        try:
            db.query("DELETE FROM files WHERE id=$id", {'id': id})
            log.warning("--> removed id %s" % id)
            return True
        except Exception, e:
            log.error(e)
            return False
    
    
    def saveFile(self, fileid, data, **kwargs):
        """
        Save the data into a file.  Return True is file successfully saved,
        otherwise False.
        
        Override this method to save files in other places.
        
        Attributes:
        fileid -- The id from the database record that corresponds to this file
        data -- The data (string of bytes) contained in the file
        
        """
        raise NotImplementedError(("You must override the implementaion of "
                                   "saveFile for your file server.  For "
                                   "example, check out the S3FileServer."))


class S3FileServer(FileServer):
    """
    In order to use this FileServer, make sure that you have the aws section
    filled out in your conf.yaml file:
    
    aws:
        access_key_id: '<ACCESS_KEY>'
        secret_access_key: '<SECRET_KEY>'
        bucket: '<BUCKET_NAME>'
    
    """
    def __init__(self, db):
        """
        Creates an Amazon AWS S3 file server wrapper.
        
        Arguments:
        ``db``
            A ``web.db.DB`` object. This may come in handy for determining a
            unique filename for a file. Will alleviate the need to make
            additional requests to the S3 server.
        
        """
        self.db = db
        
    
    def getConfigVar(self, var_name):
        return Config.get(var_name)
    
    
    def getUniqueFilename(self, filename):
        """
        Get a unique name for the file. 
        
        """
        rows = self.db.query('''select media_id from attachments
              where type in $types''', {'types':['file','image']})
        filenames = [row.media_id for row in rows]
        
        import random
        alphanum = ('1234567890'
            'abcdefghijklmnopqrstuvwxyz'
            'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
        
        # Construct random filenames until we find one that's not in use.
        while True:
            randomizer = ''.join([random.choice(alphanum) for _ in xrange(8)])
            if '-'.join([filename, randomizer]) not in filenames:
                filename = '-'.join([randomizer, filename]) if filename else randomizer
                break
        
        return filename
    
    
    def getLocalPath(self, fileid):
        """
        Get the path to the file given by the fileid on the local file system.
        This is used only to temporarily save the file before uploading it to
        the S3 server.
        
        """
        return "data/files/%s" % fileid
    
    
    def getS3Path(self, fileid):
        """
        Get the path to the file given by the fileid on the S3 server.
        
        """
        return "data/files/%s" % fileid
    
    
    def saveTemporaryLocalFile(self, path, data):
        """
        Save the file on the local file system.
        This is used only to temporarily save the file before uploading it to
        the S3 server.  Creates directory if needed.
        
        """
        directory = os.path.dirname(path)
        try:
            # Attempt to create directory structure if
            # not present.
            if not os.path.exists(directory):
                os.makedirs(directory)
                
            # Create file
            with open(path, "wb") as f:
                f.write(data)
        except Exception, e:
            log.error(e)
            return False
        
        return True
    
    
    def saveFile(self, filename, data, mirror=True, **kwargs):
        """<commit_msg>Get file paths from config instead of hard-coded "data/files" value.<commit_after>"""
    :copyright: (c) 2011 Local Projects, all rights reserved
    :license: Affero GNU GPL v3, see LICENSE for more details.
"""

import cStringIO
import traceback
import framework.util as util
import os
from framework.s3uploader import S3Uploader
from framework.log import log
from framework.controller import Controller
from framework.config import Config
from PIL import Image, ImageOps

class FileServer(object):
    """
    A generic FileServer.
    
    """
    
    def add(self, data, filename=None, make_unique=False, max_size=None, grayscale=False, mirror=True, thumb_max_size=None):
        """
        Add a file to the fileserver.  If either adding the database record for 
        the file or saving the file fails, then add will return None, and no
        modification will be made.  Otherwise, the ID of the record in the 
        database will be returned.
        
        """
        log.info("FileServer.add")
        
        # Create a unique filename
        if not filename or make_unique:
            filename = self.getUniqueFilename(filename)
        
        # Save the file to the system
        success = self.saveFile(filename, data, max_size=max_size, mirror=mirror)
        
        # Return the media_id of the file
        return filename
    
    
    def getUniqueFilename(self, filename):
        """
        Get a unique name for the file. 
        
        """
        raise NotImplementedError
    
    
    def addDbRecord(self, db, filename):
        """
        Insert a new record for a file into the given database.
        
        Arguments:
        db -- A web.py database (`web.db`) object
        app -- The name of the app (`str`)
        
        """
        try:
            id = db.insert('attachments', title=filename)
        except Exception, e:
            log.error(e)
            return None
        
        return id
    
    
    def removeDbRecord(self, db, id):
        try:
            db.query("DELETE FROM files WHERE id=$id", {'id': id})
            log.warning("--> removed id %s" % id)
            return True
        except Exception, e:
            log.error(e)
            return False
    
    
    def saveFile(self, fileid, data, **kwargs):
        """
        Save the data into a file.  Return True is file successfully saved,
        otherwise False.
        
        Override this method to save files in other places.
        
        Attributes:
        fileid -- The id from the database record that corresponds to this file
        data -- The data (string of bytes) contained in the file
        
        """
        raise NotImplementedError(("You must override the implementaion of "
                                   "saveFile for your file server.  For "
                                   "example, check out the S3FileServer."))


class S3FileServer(FileServer):
    """
    In order to use this FileServer, make sure that you have the aws section
    filled out in your conf.yaml file:
    
    aws:
        access_key_id: '<ACCESS_KEY>'
        secret_access_key: '<SECRET_KEY>'
        bucket: '<BUCKET_NAME>'
    
    """
    def __init__(self, db):
        """
        Creates an Amazon AWS S3 file server wrapper.
        
        Arguments:
        ``db``
            A ``web.db.DB`` object. This may come in handy for determining a
            unique filename for a file. Will alleviate the need to make
            additional requests to the S3 server.
        
        """
        self.db = db
        
    
    def getConfigVar(self, var_name):
        return Config.get(var_name)
    
    
    def getUniqueFilename(self, filename):
        """
        Get a unique name for the file. 
        
        """
        rows = self.db.query('''select media_id from attachments
              where type in $types''', {'types':['file','image']})
        filenames = [row.media_id for row in rows]
        
        import random
        alphanum = ('1234567890'
            'abcdefghijklmnopqrstuvwxyz'
            'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
        
        # Construct random filenames until we find one that's not in use.
        while True:
            randomizer = ''.join([random.choice(alphanum) for _ in xrange(8)])
            if '-'.join([filename, randomizer]) not in filenames:
                filename = '-'.join([randomizer, filename]) if filename else randomizer
                break
        
        return filename
    
    
    def getLocalPath(self, fileid):
        """
        Get the path to the file given by the fileid on the local file system.
        This is used only to temporarily save the file before uploading it to
        the S3 server.
        
        """
        return "%(file_path)s/%(file_id)s" % {'file_path': Config.get('media').get('file_path'),
                                             'file_id': fileid}
    
    
    def getS3Path(self, fileid):
        """
        Get the path to the file given by the fileid on the S3 server.
        
        """
        return "%(file_path)s/%(file_id)s" % {'file_path': Config.get('media').get('file_path'),
                                             'file_id': fileid}
    
    
    def saveTemporaryLocalFile(self, path, data):
        """
        Save the file on the local file system.
        This is used only to temporarily save the file before uploading it to
        the S3 server.  Creates directory if needed.
        
        """
        directory = os.path.dirname(path)
        try:
            # Attempt to create directory structure if
            # not present.
            if not os.path.exists(directory):
                os.makedirs(directory)
                
            # Create file
            with open(path, "wb") as f:
                f.write(data)
        except Exception, e:
            log.error(e)
            return False
        
        return True
    
    
    def saveFile(self, filename, data, mirror=True, **kwargs):
        """
{'fraction_non_alphanumeric': 0.08078324997880817, 'fraction_numerical': 0.004407900313639061, 'mean_word_length': 1.6734647631996373, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6853341', 'n_tokens_mistral': 3161, 'n_tokens_neox': 3085, 'n_words': 1123}