<jupyter_start><jupyter_text>(C) Crown Copyright, Met Office. All rights reserved.  stream1_volumes.ipynbCalulate the volumes used by the Stream 1 highresSST-present atmosphere only simulations<jupyter_code>import django
django.setup()
from django.db.models import Sum
from pdata_app.models import DataFile, VariableRequest
models = [
    # AWI didn't run an AMIP and so use coupled for comparison
    ('AWI-CM-1-1-LR', 'hist-1950', 'r1i1p1f002', 6.4),
    ('AWI-CM-1-1-HR', 'hist-1950', 'r1i1p1f002', 25.6),
    ('CMCC-CM2-HR4', 'highresSST-present', 'r1i1p1f1', 6.4),
    ('CMCC-CM2-VHR4', 'highresSST-present', 'r1i1p1f1', 69.6),
    ('CNRM-CM6-1', 'highresSST-present', 'r1i1p1f2', 3.2),
    ('CNRM-CM6-1-HR', 'highresSST-present', 'r1i1p1f2', 44.8),
    ('EC-Earth3P', 'highresSST-present', 'r2i1p1f1', 3.2),
    ('EC-Earth3P-HR', 'highresSST-present', 'r2i1p1f1', 44.8),
    ('ECMWF-IFS-LR', 'highresSST-present', 'r1i1p1f1', 1.3),
    ('ECMWF-IFS-HR', 'highresSST-present', 'r1i1p1f1', 10.2),
    ('MPI-ESM1-2-HR', 'highresSST-present', 'r1i1p1f1', 6.6),
    ('MPI-ESM1-2-XR', 'highresSST-present', 'r1i1p1f1', 13.1),
    ('HadGEM3-GC31-LM', 'highresSST-present', 'r1i1p1f1', 4.5),
    ('HadGEM3-GC31-HM', 'highresSST-present', 'r1i1p1f1', 64.0),
]
def _tib_to_bytes(tib):
    """Convert a value in TiB to bytes"""
    return tib * 1024**4
def _bytes_str(num_bytes):
    """
    Return a value in bytes as a value in TiB as a string
    
    :param int num_bytes: the number of bytes
    :returns: the number of bytes as a string in units of TiB to 1 decimal place
    :rtype: str
    """
    return f'{num_bytes / 1024**4:.1f}'
total_size = 0
planned_total = 0
print(f"{'Model':<20} Actual / Planned")
for model in models:
    query = {
        'data_request__climate_model__short_name': model[0],
        'data_request__experiment__short_name': model[1],
        'data_request__rip_code': model[2]
    }
    datafiles = DataFile.objects.filter(**query).distinct()
    model_size = datafiles.aggregate(Sum('size'))['size__sum']
    total_size += model_size
    planned_total += _tib_to_bytes(model[3])
    print(f'{model[0]:<20} {_bytes_str(model_size):>4} / {model[3]:>4} TiB')

print()
print(f'Stream 1 highresSST-present actual size   {_bytes_str(total_size):>5} TiB')
print(f'Stream 1 highresSST-present planned size  {_bytes_str(planned_total):>5} TiB')
print(f'Actual is {total_size / planned_total:.0%} of planned')<jupyter_output>Model                Actual / Planned
AWI-CM-1-1-LR         0.3 /  6.4 TiB
AWI-CM-1-1-HR         0.3 / 25.6 TiB
CMCC-CM2-HR4          0.8 /  6.4 TiB
CMCC-CM2-VHR4        11.6 / 69.6 TiB
CNRM-CM6-1            4.3 /  3.2 TiB
CNRM-CM6-1-HR        13.1 / 44.8 TiB
EC-Earth3P            8.4 /  3.2 TiB
EC-Earth3P-HR        32.9 / 44.8 TiB
ECMWF-IFS-LR          0.9 /  1.3 TiB
ECMWF-IFS-HR          3.3 / 10.2 TiB
MPI-ESM1-2-HR         1.5 /  6.6 TiB
MPI-ESM1-2-XR         2.8 / 13.1 TiB
HadGEM3-GC31-LM       1.8 /  4.5 TiB
HadGEM3-GC31-HM      46.4 / 64.0 TiB

Stream 1 highresSST-present actual size   128.3 TiB
Stream 1 highresSST-present planned size  303.7 TiB
Actual is 42% of planned
{'fraction_non_alphanumeric': 0.15841584158415842, 'fraction_numerical': 0.08399872245289045, 'mean_word_length': 3.030888030888031, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '43865', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1346, 'n_words': 282, 'path': 'stream1_volumes.ipynb', 'repo_name': 'PRIMAVERA-H2020/stream2-planning', 'size': '3131'}