<commit_before>import click
from multiprocessing import Process, Lock
import os
import swiftclient
import hashlib
import json
import time


@click.command()
@click.option('--filename', help='File to be uploaded.', required=True)
@click.option('--segment_size', default=1,
              help='Size of segments the file will be divided into in '
              'megabytes. Default and minimum is 1MB')
@click.option('--container', help='Container to place the file.',
              required=True)
@click.option('--auth_token', help='Swift auth token from swift stat.')
@click.option('--storage_url', help='Storage url found from swift stat -v.')
def slo_upload(filename, segment_size, container, auth_token, storage_url):
    """Given the swift credentials, upload the targeted file onto swift as a
    Static Large Object"""

    # Check credentials
    (auth_token, storage_url) = validate_credentials(storage_url, auth_token,
                                                     container)

    #TODO: Check for existing tempmanifest.json

    # Variables required by several functions wrapped in a dictionary for
    # convenience.
    args = {
        "filename": filename,
        "segment_size": segment_size,
        "container": container,
        "auth_token": auth_token,
        "storage_url": storage_url,
        "lock": Lock(),
    }

    processes = []  # Holder for processes

    segment_counter = 1  # Counter for segments created.

    max_processes = 10  # Maximum number of processes

    with open(filename, "r") as f:
        while True:

            while len(processes) > max_processes:
                p = processes.pop()
                p.join()

            buf = f.read(int(segment_size * 1048576))
            if not buf:
                 # we've read the entire file in, so we're done.
                break

            segment_name = "{}".format(
                "%08d" % segment_counter
            )

            # Create file
            segment = open(segment_name, "w")
            segment.write(buf)
            segment.close()

            # The location segments will be stored on swift is within a pseudo
            # folder.
            swift_destination = os.path.join(
                filename.split("/")[-1] + "_segments", segment_name)

            # Upload and delete the segment.
            p = Process(target=process_segment,
                        args=(args, segment_name, swift_destination))
            p.start()
            processes = [p] + processes

            segment_counter += 1
    f.close()

    while len(processes) > 0:
        p = processes.pop()
        p.join()

    # Create manifest file
    create_manifest_file("manifest.json", container)

    # Upload manifest file
    upload_manifest_file("manifest.json", args)

    delete_file("upload_cache")
    delete_file("manifest.json")


def validate_credentials(storage_url, auth_token, container):
    '''Validate credentials. Check to make sure auth token and storage url
    work for the given tenant. If they are not set, check that the required
    os variables are set and that the work. Return valid auth token and storage
    url. Exit program if credentials are invalid or os variables not set.'''

    # Check given credentials
    if not (storage_url or auth_token):

        # Check OS variables
        if (not os.environ.get("OS_AUTH_URL") or
                not os.environ.get("OS_USERNAME") or
                not os.environ.get("OS_PASSWORD") or
                not os.environ.get("OS_TENANT_NAME")):

            # Exit if variables are not set.
            click.echo("Please pass in --storage_url and --auth_token or make"<commit_msg>Check for existing upload_cache and continue upload from that point. Closes Issue #4<commit_after>import click
from multiprocessing import Process, Lock
import os
import swiftclient
import hashlib
import json
import time


@click.command()
@click.option('--filename', help='File to be uploaded.', required=True)
@click.option('--segment_size', default=1,
              help='Size of segments the file will be divided into in '
              'megabytes. Default and minimum is 1MB')
@click.option('--container', help='Container to place the file.',
              required=True)
@click.option('--auth_token', help='Swift auth token from swift stat.')
@click.option('--storage_url', help='Storage url found from swift stat -v.')
def slo_upload(filename, segment_size, container, auth_token, storage_url):
    """Given the swift credentials, upload the targeted file onto swift as a
    Static Large Object"""

    # Check credentials
    (auth_token, storage_url) = validate_credentials(storage_url, auth_token,
                                                     container)

    #TODO: Check for existing tempmanifest.json

    # Variables required by several functions wrapped in a dictionary for
    # convenience.
    args = {
        "filename": filename,
        "segment_size": segment_size,
        "container": container,
        "auth_token": auth_token,
        "storage_url": storage_url,
        "lock": Lock(),
    }

    processes = []  # Holder for processes

    segment_counter = 1  # Counter for segments created.

    max_processes = 10  # Maximum number of processes

    with open(filename, "r") as f:

        # Check if upload_cache exists:
        try:
            open("upload_cache")
            segment_counter = fast_forward_file(f, segment_size)
            f.seek(segment_size * 1048576 * (segment_counter - 1))
        except IOError:
            pass
        while True:

            while len(processes) >= max_processes:
                p = processes.pop()
                p.join()

            buf = f.read(int(segment_size * 1048576))
            if not buf:
                 # we've read the entire file in, so we're done.
                break

            segment_name = "{}".format(
                "%08d" % segment_counter
            )

            # Create file
            segment = open(segment_name, "w")
            segment.write(buf)
            segment.close()

            # The location segments will be stored on swift is within a pseudo
            # folder.
            swift_destination = os.path.join(
                filename.split("/")[-1] + "_segments", segment_name)

            # Upload and delete the segment.
            p = Process(target=process_segment,
                        args=(args, segment_name, swift_destination))
            p.start()
            processes = [p] + processes

            segment_counter += 1
    f.close()

    while len(processes) > 0:
        p = processes.pop()
        p.join()

    # Create manifest file
    create_manifest_file("manifest.json", container)

    # Upload manifest file
    upload_manifest_file("manifest.json", args)

    delete_file("upload_cache")
    delete_file("manifest.json")


def fast_forward_file(file, segment_size):
    '''Find the last sequential segment name from upload_cache. Fast forward
    the given file to the corresponding position for the next segment. Return
    the next segment number.'''

    segments = []

    cache = open('upload_cache', "r")
    for line in cache:
        segments.append(int(line.split(":")[0]))

    segments.sort()

    i = 1

    while i < len(segments):
        if segments[i - 1] != segments[i] - 1:
            return segments[i - 1] + 1
        i += 1

    return segments[-1] + 1


def validate_credentials(storage_url, auth_token, container):
    '''Validate credentials. Check to make sure auth token and storage url
    work for the given tenant. If they are not set, check that the required
    os variables are set and that the work. Return valid auth token and storage
    url. Exit program if credentials are invalid or os variables not set.'''

    # Check given credentials
    if not (storage_url or auth_token):

        # Check OS variables
        if (not os.environ.get("OS_AUTH_URL") or
                not os.environ.get("OS_USERNAME") or
                not os.environ.get("OS_PASSWORD") or
                not os.environ.get("OS_TENANT_NAME")):

            # Exit if variables are not set.
            click.echo("Please pass in --storage_url and --auth_token or make"
{'fraction_non_alphanumeric': 0.09400048579062424, 'fraction_numerical': 0.006315278115132378, 'mean_word_length': 2.216796875, 'pattern_counts': {'":': 13, '<': 4, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3044147', 'n_tokens_mistral': 2269, 'n_tokens_neox': 2139, 'n_words': 729}