<commit_before>'''

This is a thin veneer around a pyfilesystem. It adds a few bits of
functionality:

1) Django configuration. This can go to Amazon S3 or a static
filesystem.
2) The ability to get URLs for objects stored on the filesystem.
3) The ability to create objects with a limited lifetime. A 
task can garbage-collect those objects. 

'''

import json
import os
import os.path
import types

from django.conf import settings

from models import FSExpirations


if hasattr(settings, 'DJFS'):
    djfs_settings = settings.DJFS
else:
    djfs_settings = {'type' : 'osfs',
                     'directory_root' : 'django-pyfs/static/django-pyfs', 
                     'url_root' : '/static/django-pyfs'}

if djfs_settings['type'] == 'osfs':
    from fs.osfs import OSFS
elif djfs_settings['type'] == 's3fs':
    from fs.s3fs import S3FS
    from boto.s3.connection import S3Connection
    from boto.s3.key import Key
    s3conn = S3Connection()
else: 
    raise AttributeError("Bad filesystem: "+str(djfs_settings['type']))

def get_filesystem(namespace):
    ''' Returns a pyfilesystem for static module storage. 

    The file system will have two additional properties: 
    1) get_url: A way to get a URL for a static file download
    2) expire: A way to expire files (so they are automatically destroyed)
    '''
    if djfs_settings['type'] == 'osfs':
        return get_osfs( namespace )
    elif djfs_settings['type'] == 's3fs':
        return get_s3fs( namespace )<commit_msg>Adds support for auth. Note: UNTESTED (but will be in a few minutes, so if you see this later, tested)<commit_after>'''

This is a thin veneer around a pyfilesystem. It adds a few bits of
functionality:

1) Django configuration. This can go to Amazon S3 or a static
filesystem.
2) The ability to get URLs for objects stored on the filesystem.
3) The ability to create objects with a limited lifetime. A 
task can garbage-collect those objects. 

'''

import json
import os
import os.path
import types

from django.conf import settings

from models import FSExpirations


if hasattr(settings, 'DJFS'):
    djfs_settings = settings.DJFS
else:
    djfs_settings = {'type' : 'osfs',
                     'directory_root' : 'django-pyfs/static/django-pyfs', 
                     'url_root' : '/static/django-pyfs'}

if djfs_settings['type'] == 'osfs':
    from fs.osfs import OSFS
elif djfs_settings['type'] == 's3fs':
    from fs.s3fs import S3FS
    from boto.s3.connection import S3Connection
    from boto.s3.key import Key
    key_id = djfs_settings.get('AWS_ACCESS_KEY_ID', None)
    key_secret = djfs_settings.get('AWS_SECRET_ACCESS_KEY', None)
    s3conn = S3Connection(aws_access_key_id=key_id, aws_secret_access_key=key_secret)
else: 
    raise AttributeError("Bad filesystem: "+str(djfs_settings['type']))

def get_filesystem(namespace):
    ''' Returns a pyfilesystem for static module storage. 

    The file system will have two additional properties: 
    1) get_url: A way to get a URL for a static file download
    2) expire: A way to expire files (so they are automatically destroyed)
    '''
    if djfs_settings['type'] == 'osfs':
        return get_osfs( namespace )
    elif djfs_settings['type'] == 's3fs':
        return get_s3fs( namespace )
{'fraction_non_alphanumeric': 0.0912442396313364, 'fraction_numerical': 0.009831029185867896, 'mean_word_length': 3.4972375690607733, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7413458', 'n_tokens_mistral': 1041, 'n_tokens_neox': 998, 'n_words': 364}