<commit_before>#!/usr/bin/env python
""" md5s3stash
    content addressable storage in AWS S3
"""
from __future__ import unicode_literals
import sys
import os
import argparse
import tempfile
import urllib2
import urllib
import logging
import shutil
import hashlib
import basin
import boto
import urlparse
from collections import namedtuple


def main(argv=None):
    parser = argparse.ArgumentParser(
        description='content addressable storage in AWS S3')
    parser.add_argument('url', nargs='+',
                        help='URL or path of source file to stash')
    parser.add_argument('-b', '--bucket_base', nargs="?",
                        help='this must be a unique name in all of AWS S3')
    parser.add_argument('-t', '--tempdir', required=False,
                        help="if your files might be large, make sure this is on a big disk")
    parser.add_argument('-w', '--warnings', default=False,
                        help='show python `DeprecationWarning`s supressed by default',
                        required=False, action='store_true')
    parser.add_argument('--loglevel', default='ERROR', required=False)

    if argv is None:
        argv = parser.parse_args()

    if argv.bucket_base:
        bucket_base = argv.bucket_base[0]
    else:
        assert 'BUCKET_BASE' in os.environ, "`-b` or `BUCKET_BASE` must be set"
        bucket_base = os.environ['BUCKET_BASE']

    if not argv.warnings:
        # supress warnings
        # http://stackoverflow.com/a/2047600/1763984
        import warnings
        warnings.simplefilter("ignore", DeprecationWarning)

    if argv.tempdir:
        tempfile.tempdir = argv.tempdir

    # set debugging level
    numeric_level = getattr(logging, argv.loglevel.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % argv.loglevel)
    logging.basicConfig(level=numeric_level, )

    # if being used in a library, probably want to be able to recycle
    # connection?
    conn = boto.connect_s3()
    for url in argv.url:
        print("{0}\t{1}\t{2}\t{3}".format(
            *md5s3stash(url, bucket_base, conn)
        ))


def md5s3stash(url, bucket_base, conn=None):
    """ stash a file at `url` in the named `bucket_base` ,
        `conn` is an optional boto.connect_s3()
    """
    (inputfile, tdir, baseFile, md5, mime_type) = checkChunks(url)
    s3_url = md5_to_s3_url(md5, bucket_base)
    temp_file = os.path.join(tdir, baseFile)
    if conn is None:
        conn = boto.connect_s3()
    s3move(temp_file, s3_url, mime_type, conn)
    shutil.rmtree(tdir)
    StashReport = namedtuple('StashReport', 'url, md5, s3_url, mime_type')
    report = StashReport(url, md5, s3_url, mime_type)
    logging.getLogger('MD5S3:stash').info(report)
    return report


def md5_to_s3_url(md5, bucket_base):
    """ calculate the s3 URL given an md5 and an bucket_base """
    return "s3://{0}.{1}/{2}".format(
        md5_to_bucket_shard(md5),
        bucket_base,
        md5
    )


def md5_to_bucket_shard(md5):
    """ calculate the shard label of the bucket name from md5 """
    # "Consider utilizing multiple buckets that start with different
    # alphanumeric characters. This will ensure a degree of partitioning
    # from the start. The higher your volume of concurrent PUT and
    # GET requests, the more impact this will likely have."
    #  -- http://aws.amazon.com/articles/1904
    # "Bucket names must be a series of one or more labels. Adjacent
    # labels are separated by a single period (.). [...] Each label must 
    # start and end with a lowercase letter or a number. "
    #  -- http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
    # see also:  http://en.wikipedia.org/wiki/Base_36
    ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyz"
    # http://stats.stackexchange.com/a/70884/14900
    # take the first two digits of the hash and turn that into an inteter
    # this should be evenly distributed
    int_value = int(md5[0], 16)+10*int(md5[1], 16)
    # divide by the length of the alphabet and take the remainder<commit_msg>allowing http access to backing buckets for now<commit_after>#!/usr/bin/env python
""" md5s3stash
    content addressable storage in AWS S3
"""
from __future__ import unicode_literals
import sys
import os
import argparse
import tempfile
import urllib2
import urllib
import logging
import shutil
import hashlib
import basin
import boto
import urlparse
from collections import namedtuple


def main(argv=None):
    parser = argparse.ArgumentParser(
        description='content addressable storage in AWS S3')
    parser.add_argument('url', nargs='+',
                        help='URL or path of source file to stash')
    parser.add_argument('-b', '--bucket_base', nargs="?",
                        help='this must be a unique name in all of AWS S3')
    parser.add_argument('-t', '--tempdir', required=False,
                        help="if your files might be large, make sure this is on a big disk")
    parser.add_argument('-w', '--warnings', default=False,
                        help='show python `DeprecationWarning`s supressed by default',
                        required=False, action='store_true')
    parser.add_argument('--loglevel', default='ERROR', required=False)

    if argv is None:
        argv = parser.parse_args()

    if argv.bucket_base:
        bucket_base = argv.bucket_base[0]
    else:
        assert 'BUCKET_BASE' in os.environ, "`-b` or `BUCKET_BASE` must be set"
        bucket_base = os.environ['BUCKET_BASE']

    if not argv.warnings:
        # supress warnings
        # http://stackoverflow.com/a/2047600/1763984
        import warnings
        warnings.simplefilter("ignore", DeprecationWarning)

    if argv.tempdir:
        tempfile.tempdir = argv.tempdir

    # set debugging level
    numeric_level = getattr(logging, argv.loglevel.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % argv.loglevel)
    logging.basicConfig(level=numeric_level, )

    # if being used in a library, probably want to be able to recycle
    # connection?
    conn = boto.connect_s3()
    for url in argv.url:
        print("{0}\t{1}\t{2}\t{3}".format(
            *md5s3stash(url, bucket_base, conn)
        ))


def md5s3stash(url, bucket_base, conn=None):
    """ stash a file at `url` in the named `bucket_base` ,
        `conn` is an optional boto.connect_s3()
    """
    (inputfile, tdir, baseFile, md5, mime_type) = checkChunks(url)
    s3_url = md5_to_s3_url(md5, bucket_base)
    temp_file = os.path.join(tdir, baseFile)
    if conn is None:
        conn = boto.connect_s3()
    s3move(temp_file, s3_url, mime_type, conn)
    shutil.rmtree(tdir)
    StashReport = namedtuple('StashReport', 'url, md5, s3_url, mime_type')
    report = StashReport(url, md5, s3_url, mime_type)
    logging.getLogger('MD5S3:stash').info(report)
    return report


# think about refactoring the next two functions

def md5_to_s3_url(md5, bucket_base):
    """ calculate the s3 URL given an md5 and an bucket_base """
    return "s3://{0}.{1}/{2}".format(
        md5_to_bucket_shard(md5),
        bucket_base,
        md5
    )


def md5_to_http_url(md5, bucket_base):
    """ calculate the http URL given an md5 and an bucket_base """
    return "http://s3.amazonaws.com/{0}.{1}/{2}".format(
        md5_to_bucket_shard(md5),
        bucket_base,
        md5
    )


def md5_to_bucket_shard(md5):
    """ calculate the shard label of the bucket name from md5 """
    # "Consider utilizing multiple buckets that start with different
    # alphanumeric characters. This will ensure a degree of partitioning
    # from the start. The higher your volume of concurrent PUT and
    # GET requests, the more impact this will likely have."
    #  -- http://aws.amazon.com/articles/1904
    # "Bucket names must be a series of one or more labels. Adjacent
    # labels are separated by a single period (.). [...] Each label must 
    # start and end with a lowercase letter or a number. "
    #  -- http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
    # see also:  http://en.wikipedia.org/wiki/Base_36
    ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyz"
    # http://stats.stackexchange.com/a/70884/14900
    # take the first two digits of the hash and turn that into an inteter
    # this should be evenly distributed
    int_value = int(md5[0], 16)+10*int(md5[1], 16)
    # divide by the length of the alphabet and take the remainder
{'fraction_non_alphanumeric': 0.10537432527575687, 'fraction_numerical': 0.023938042712978173, 'mean_word_length': 2.9531539888682747, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5645123', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2691, 'n_words': 793}