<commit_before>from azure.storage.blob import BlockBlobService
from azure.storage.blob import PublicAccess
from azure.storage.blob import ContentSettings
import os.path

def upload2(path):
    files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and f.endswith('.tar.gz')]
    for f in files:
        print(f)


# BEFORE RUNNING: pip install azure-storage
# container is the name for the Azure blog storage container to create.
# path is the path to the directory containing .tar.gz files to upload.
def upload(path, container, account, key):
    block_blob_service = BlockBlobService(account_name=account, account_key=key)
    block_blob_service.create_container(container, public_access=PublicAccess.Container)
    files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and f.endswith('.tar.gz')]
    for file in files:
        print("Uploading " + file)
        block_blob_service.create_blob_from_path(
            container,
            file,   # This is the path to the file on the local machine.
            file)   # TODO: This is the name for the blog on azure. Really want to strip off the first part of the path.

<commit_msg>Fixes to Azure upload code.<commit_after>from azure.storage.blob import BlockBlobService
from azure.storage.blob import PublicAccess
from azure.storage.blob import ContentSettings
import os.path


# BEFORE RUNNING: pip install azure-storage
# container is the name for the Azure blog storage container to create.
# path is the path to the directory containing .tar.gz files to upload.
def upload(path, container, account, key):
    print("Creating blob container " + container)
    block_blob_service = BlockBlobService(account_name=account, account_key=key)
    block_blob_service.create_container(container, public_access=PublicAccess.Container)
    files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and f.endswith('.tar.gz')]
    for file in files:
        blob = os.path.join("chunks", file)
        print("Uploading " + os.path.join(path, file) + " to " + blob)
        block_blob_service.create_blob_from_path(
            container,
            blob,
            file)


{'fraction_non_alphanumeric': 0.08995433789954338, 'fraction_numerical': 0.00045662100456621003, 'mean_word_length': 3.6223628691983123, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2937468', 'n_tokens_mistral': 650, 'n_tokens_neox': 642, 'n_words': 212}