<commit_before>#!/usr/bin/env python
""" md5s3stash
    content addressable storage in AWS S3
"""
from __future__ import unicode_literals
import sys
import os
import argparse
import tempfile
import urllib2
import urllib
import urlparse
import base64
import logging
import hashlib
import basin
import boto
from collections import namedtuple


def main(argv=None):
    parser = argparse.ArgumentParser(
        description='content addressable storage in AWS S3')
    parser.add_argument('url', nargs='+',
                        help='URL or path of source file to stash')
    parser.add_argument('-b', '--bucket_base', nargs="?",
                        help='this must be a unique name in all of AWS S3')
    parser.add_argument(
        '-t', '--tempdir',
        required=False,
        help="if your files might be large, make sure this is on a big disk"
    )
    parser.add_argument(
        '-w', '--warnings',
        default=False,
        help='show python `DeprecationWarning`s supressed by default',
        required=False,
        action='store_true',
    )
    parser.add_argument('--loglevel', default='ERROR', required=False)
    parser.add_argument('-u', '--username', required=False,
                        help='username for downloads requiring BasicAuth')
    parser.add_argument('-p', '--password', required=False,
                        help='password for downloads requiring BasicAuth')

    if argv is None:
        argv = parser.parse_args()

    if argv.bucket_base:
        bucket_base = argv.bucket_base[0]<commit_msg>bug in -b --bucket_base parsing from CLI<commit_after>#!/usr/bin/env python
""" md5s3stash
    content addressable storage in AWS S3
"""
from __future__ import unicode_literals
import sys
import os
import argparse
import tempfile
import urllib2
import urllib
import urlparse
import base64
import logging
import hashlib
import basin
import boto
from collections import namedtuple


def main(argv=None):
    parser = argparse.ArgumentParser(
        description='content addressable storage in AWS S3')
    parser.add_argument('url', nargs='+',
                        help='URL or path of source file to stash')
    parser.add_argument('-b', '--bucket_base', nargs="?",
                        help='this must be a unique name in all of AWS S3')
    parser.add_argument(
        '-t', '--tempdir',
        required=False,
        help="if your files might be large, make sure this is on a big disk"
    )
    parser.add_argument(
        '-w', '--warnings',
        default=False,
        help='show python `DeprecationWarning`s supressed by default',
        required=False,
        action='store_true',
    )
    parser.add_argument('--loglevel', default='ERROR', required=False)
    parser.add_argument('-u', '--username', required=False,
                        help='username for downloads requiring BasicAuth')
    parser.add_argument('-p', '--password', required=False,
                        help='password for downloads requiring BasicAuth')

    if argv is None:
        argv = parser.parse_args()

    if argv.bucket_base:
        bucket_base = argv.bucket_base
{'fraction_non_alphanumeric': 0.10352564102564103, 'fraction_numerical': 0.0054487179487179484, 'mean_word_length': 2.751201923076923, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3043951', 'n_tokens_mistral': 885, 'n_tokens_neox': 854, 'n_words': 273}