<jupyter_start><jupyter_text>Read and write from Numpy to Azure Data Lake and Azure Blob storage  Initialize public Blob filesystem<jupyter_code>container_name = "datasets"
storage_options = {"account_name": "azuremlexamples"}
from adlfs import AzureBlobFileSystem as abfs

fs = abfs(**storage_options)
files = fs.ls(f"{container_name}/mnist")
files<jupyter_output><empty_output><jupyter_text>Define fuctions to read gzipped MNIST data<jupyter_code>import numpy as np


def read_images(f, num_images, image_size=28):
    f.read(16)  # magic

    buf = f.read(image_size * image_size * num_images)
    images = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)
    images = images.reshape(num_images, image_size, image_size, 1)

    return images


def read_labels(f, num_labels):
    f.read(8)  # magic

    buf = f.read(num_labels)
    labels = np.frombuffer(buf, dtype=np.uint8)

    return labels<jupyter_output><empty_output><jupyter_text>Read in gzipped MNIST data<jupyter_code>import gzip

train_len = 60000
test_len = 10000

for f in files:
    if "train-images" in f:
        X_train = read_images(gzip.open(fs.open(f)), train_len)
    elif "train-labels" in f:
        y_train = read_labels(gzip.open(fs.open(f)), train_len)
    elif "images" in f:
        X_test = read_images(gzip.open(fs.open(f)), test_len)
    elif "labels" in f:
        y_test = read_labels(gzip.open(fs.open(f)), test_len)<jupyter_output><empty_output><jupyter_text>Verify expected results<jupyter_code>from random import randint

i = randint(0, train_len)
x = X_train[i]
y = y_train[i]
import matplotlib.pyplot as plt

plt.imshow(x.squeeze())
plt.title(f"Label: {y}")<jupyter_output><empty_output><jupyter_text>Initialize private Blob filesystem<jupyter_code>from azureml.core import Workspace

ws = Workspace.from_config()
ds = ws.datastores["workspaceblobstore"]

container_name = ds.container_name
storage_options = {"account_name": ds.account_name, "account_key": ds.account_key}
fs = abfs(**storage_options)
fs
fs.ls(f"{container_name}")<jupyter_output><empty_output><jupyter_text>Write numpy arrays using `np.save`<jupyter_code>with fs.open(f"{container_name}/example-data/mnist/X_train.npy", "wb") as f:
    np.save(f, X_train)

with fs.open(f"{container_name}/example-data/mnist/y_train.npy", "wb") as f:
    np.save(f, y_train)

with fs.open(f"{container_name}/example-data/mnist/X_test.npy", "wb") as f:
    np.save(f, X_test)

with fs.open(f"{container_name}/example-data/mnist/y_test.npy", "wb") as f:
    np.save(f, y_test)<jupyter_output><empty_output><jupyter_text>Load numpy arrays using `np.load`<jupyter_code>with fs.open(f"{container_name}/example-data/mnist/X_train.npy", "rb") as f:
    X_train = np.load(f)

with fs.open(f"{container_name}/example-data/mnist/y_train.npy", "rb") as f:
    y_train = np.load(f)

with fs.open(f"{container_name}/example-data/mnist/X_test.npy", "rb") as f:
    X_test = np.load(f)

with fs.open(f"{container_name}/example-data/mnist/y_test.npy", "rb") as f:
    y_test = np.load(f)<jupyter_output><empty_output><jupyter_text>Verify expected results<jupyter_code>from random import randint

i = randint(0, train_len)
x = X_train[i]
y = y_train[i]
import matplotlib.pyplot as plt

plt.imshow(x.squeeze())
plt.title(f"Label: {y}")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13770094024871096, 'fraction_numerical': 0.006672732787382469, 'mean_word_length': 5.504930966469428, 'pattern_counts': {'":': 3, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '202560', 'license': 'MIT', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1297, 'n_words': 235, 'path': 'notebooks/blob-adls-numpy/1.read-write.ipynb', 'repo_name': 'shbijlan/azureml-examples', 'size': '3297'}