<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
gifshare - Share Gifs via Amazon S3
"""

FOOTER = """
Copyright (c) 2014 by <NAME>.
MIT Licensed, see LICENSE.txt for more details.
"""

__version__ = '0.0.1'

import argparse
from ConfigParser import SafeConfigParser
import logging
from os.path import expanduser, isfile, basename, splitext
import random
import re
from StringIO import StringIO
import sys

from boto.s3.key import Key
from boto.s3.connection import S3Connection

import magic
import progressbar
import requests


class UserException(Exception):
    """
    Indicates that the user should not see a stack-trace - just the
    exception message.
    """
    pass


class UnknownFileType(UserException):
    pass


class FileAlreadyExists(UserException):
    pass


URL_RE = re.compile(r'^http.*')
CONTENT_TYPE_MAP = {
    'gif': 'image/gif',
    'jpeg': 'image/jpeg',
    'png': 'image/png',
}
LOG = logging.getLogger('gifshare')


def correct_ext(data, is_buffer=False):
    magic_output = magic.from_buffer(data) if is_buffer else magic.from_file(
        data)
    match = re.search(r'JPEG|GIF|PNG', magic_output)
    if match:
        return match.group(0).lower()
    else:
        raise UnknownFileType("Unknown file type: {}".format(magic_output))


def load_config():
    config = SafeConfigParser()
    config.read([expanduser('~/.gifshare'), '.gifshare'])
    return config


def download_file(url):
    LOG.debug("Downloading image ...")
    response = requests.get(url, stream=True)
    length = int(response.headers['content-length'])
    LOG.debug('Content length: %d', length)
    content = StringIO()
    i = 0
    widgets = ['Downloading image ', progressbar.Bar(), progressbar.Percentage()]
    pbar = progressbar.ProgressBar(widgets=widgets, maxval=length).start()
    for chunk in response.iter_content(64):
        i += len(chunk)
        LOG.debug('Update: %d', i)
        content.write(chunk)
        pbar.update(i)
    pbar.finish()

    return content.getvalue()


def get_name_from_url(url):
    return re.match(r'.*/([^/\.]+)', url).group(1)


def upload_callback():
    pbar = [None]
    widgets = ['Uploading image ', progressbar.Bar(), progressbar.Percentage()]

    def callback(update, total):
        if pbar[0] is None:
            pbar[0] = progressbar.ProgressBar(widgets=widgets, maxval=total)
            pbar[0].start()
        else:
            pbar[0].update(update)
        if update == total:
            pbar[0].finish()

    return callback


def upload_url(config, url, name=None, force=False):
    LOG.debug('Uploading URL ...')
    data = download_file(url)
    ext = correct_ext(data, True)
    filename = (name or get_name_from_url(url)) + '.' + ext
    dest_url = config.get('default', 'web_root') + filename
    bucket = Bucket(config)
    key = bucket.key_for(filename, CONTENT_TYPE_MAP[ext])
    if key.exists() and not force:
        raise FileAlreadyExists("File at {} already exists!".format(dest_url))
    LOG.debug("Uploading image ...")

    key.set_contents_from_string(data, cb=upload_callback())

    return dest_url


def upload_file(config, path, name=None, force=False):
    LOG.debug("Uploading file ...")
    ext = correct_ext(path)
    filename = (name or splitext(basename(path))[0]) + '.' + ext
    url = config.get('default', 'web_root') + filename
    bucket = Bucket(config)
    key = bucket.key_for(filename, CONTENT_TYPE_MAP[ext])
    if key.exists() and not force:
        raise FileAlreadyExists("File at {} already exists!".format(url))
    key.set_contents_from_filename(path, cb=upload_callback())

    return url


class Bucket(object):
    def __init__(self, config):
        self._bucket = None
        self._key_id = config.get('default', 'aws_access_id')
        self._access_key = config.get('default', 'aws_secret_access_key')
        self._bucket_name = config.get('default', 'bucket')
        self._web_root = config.get('default', 'web_root')

    @property
    def bucket(self):
        if not self._bucket:
            conn = S3Connection(self._key_id, self._access_key)
            self._bucket = conn.get_bucket(self._bucket_name)
        return self._bucket

    def key_for(self, filename, content_type):
        k = Key(self.bucket, filename)
        k.content_type = content_type
        return k

    def list(self):
        bucket = self.bucket
        for key in bucket.list():
            url = self._web_root + key.name
            yield url


def command_upload(arguments, config):
    path = arguments.path
    if not URL_RE.match(path):
        if isfile(path):
            print upload_file(
                config, path, arguments.key, force=arguments.force)
        else:
            raise IOError(
                '{} does not exist or is not a file!'.format(path))
    else:
        print upload_url(
            config, path, arguments.key, force=arguments.force)


def command_list(arguments, config):
    bucket = Bucket(config)
    if not arguments.random:
        for item in bucket.list():
            print item
    else:
        print random.choice(list(bucket.list()))


def main(argv=sys.argv[1:]):
    try:
        a_parser = argparse.ArgumentParser(description=__doc__, epilog=FOOTER)
        a_parser.add_argument(
            '--version',
            action='version',
            version='%(prog)s ' + __version__)

        a_parser.add_argument(
            '-v', '--verbose',
            action='store_true',
            help='print out more stuff')
<commit_msg>Refactored upload functions into Bucket.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
gifshare - Share Gifs via Amazon S3
"""

FOOTER = """
Copyright (c) 2014 by <NAME>.
MIT Licensed, see LICENSE.txt for more details.
"""

__version__ = '0.0.1'

import argparse
from ConfigParser import SafeConfigParser
import logging
from os.path import expanduser, isfile, basename, splitext
import random
import re
from StringIO import StringIO
import sys

from boto.s3.key import Key
from boto.s3.connection import S3Connection

import magic
import progressbar
import requests


class UserException(Exception):
    """
    Indicates that the user should not see a stack-trace - just the
    exception message.
    """
    pass


class UnknownFileType(UserException):
    pass


class FileAlreadyExists(UserException):
    pass


URL_RE = re.compile(r'^http.*')
CONTENT_TYPE_MAP = {
    'gif': 'image/gif',
    'jpeg': 'image/jpeg',
    'png': 'image/png',
}
LOG = logging.getLogger('gifshare')


def correct_ext(data, is_buffer=False):
    magic_output = magic.from_buffer(data) if is_buffer else magic.from_file(
        data)
    match = re.search(r'JPEG|GIF|PNG', magic_output)
    if match:
        return match.group(0).lower()
    else:
        raise UnknownFileType("Unknown file type: {}".format(magic_output))


def load_config():
    config = SafeConfigParser()
    config.read([expanduser('~/.gifshare'), '.gifshare'])
    return config


def download_file(url):
    LOG.debug("Downloading image ...")
    response = requests.get(url, stream=True)
    length = int(response.headers['content-length'])
    LOG.debug('Content length: %d', length)
    content = StringIO()
    i = 0
    widgets = ['Downloading image ', progressbar.Bar(), progressbar.Percentage()]
    pbar = progressbar.ProgressBar(widgets=widgets, maxval=length).start()
    for chunk in response.iter_content(64):
        i += len(chunk)
        LOG.debug('Update: %d', i)
        content.write(chunk)
        pbar.update(i)
    pbar.finish()

    return content.getvalue()


def get_name_from_url(url):
    return re.match(r'.*/([^/\.]+)', url).group(1)


def upload_callback():
    pbar = [None]
    widgets = ['Uploading image ', progressbar.Bar(), progressbar.Percentage()]

    def callback(update, total):
        if pbar[0] is None:
            pbar[0] = progressbar.ProgressBar(widgets=widgets, maxval=total)
            pbar[0].start()
        else:
            pbar[0].update(update)
        if update == total:
            pbar[0].finish()

    return callback





class Bucket(object):
    def __init__(self, config):
        self._bucket = None
        self._key_id = config.get('default', 'aws_access_id')
        self._access_key = config.get('default', 'aws_secret_access_key')
        self._bucket_name = config.get('default', 'bucket')
        self._web_root = config.get('default', 'web_root')

    @property
    def bucket(self):
        if not self._bucket:
            conn = S3Connection(self._key_id, self._access_key)
            self._bucket = conn.get_bucket(self._bucket_name)
        return self._bucket

    def key_for(self, filename, content_type):
        k = Key(self.bucket, filename)
        k.content_type = content_type
        return k

    def list(self):
        bucket = self.bucket
        for key in bucket.list():
            url = self._web_root + key.name
            yield url

    def upload_url(self, url, name=None, force=False):
        LOG.debug('Uploading URL ...')
        data = download_file(url)
        ext = correct_ext(data, True)
        filename = (name or get_name_from_url(url)) + '.' + ext
        dest_url = self._web_root + filename
        key = self.key_for(filename, CONTENT_TYPE_MAP[ext])
        if key.exists() and not force:
            raise FileAlreadyExists(
                "File at {} already exists!".format(dest_url))
        LOG.debug("Uploading image ...")
        key.set_contents_from_string(data, cb=upload_callback())

        return dest_url

    def upload_file(self, path, name=None, force=False):
        LOG.debug("Uploading file ...")
        ext = correct_ext(path)
        filename = (name or splitext(basename(path))[0]) + '.' + ext
        url = self._web_root + filename
        key = self.key_for(filename, CONTENT_TYPE_MAP[ext])
        if key.exists() and not force:
            raise FileAlreadyExists("File at {} already exists!".format(url))
        LOG.debug("Uploading image ...")
        key.set_contents_from_filename(path, cb=upload_callback())

        return url


def command_upload(arguments, config):
    path = arguments.path
    if not URL_RE.match(path):
        if isfile(path):
            print Bucket(config).upload_file(
                path, arguments.key, force=arguments.force)
        else:
            raise IOError(
                '{} does not exist or is not a file!'.format(path))
    else:
        print Bucket(config).upload_url(
            path, arguments.key, force=arguments.force)


def command_list(arguments, config):
    bucket = Bucket(config)
    if not arguments.random:
        for item in bucket.list():
            print item
    else:
        print random.choice(list(bucket.list()))


def main(argv=sys.argv[1:]):
    try:
        a_parser = argparse.ArgumentParser(description=__doc__, epilog=FOOTER)
        a_parser.add_argument(
            '--version',
            action='version',
            version='%(prog)s ' + __version__)

        a_parser.add_argument(
            '-v', '--verbose',
            action='store_true',
            help='print out more stuff')

{'fraction_non_alphanumeric': 0.12568502380738478, 'fraction_numerical': 0.0044919593926870905, 'mean_word_length': 2.755735492577598, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6758733', 'n_tokens_mistral': 3474, 'n_tokens_neox': 3355, 'n_words': 809}