<jupyter_start><jupyter_text>How compressed are the layers?<jupyter_code>import numpy as np    
import zlib

def compression_ratio_np(A):
    B = zlib.compress(A, 1)
    BZ = zlib.compress(A,0)
    return len(B)/len(BZ)

def compression_ratio(T):
    return compression_ratio_np(T.cpu().numpy())

def byte_size(a):
    return a.element_size() * a.nelement()

import pandas as pd
D=foo['model_state_dict']
rows=[]
for x in D:
    a=D[x]
    rows.append([x, a.shape, byte_size(a), compression_ratio(a)])
df_compression=pd.DataFrame(rows, columns=['Parameter', 'Shape', 'Size', 'Compression Ratio'])
with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
    display(df_compression)
df_compression['Size'].sum()/1000000
64*64*3*3
147456/36864<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14681107099879662, 'fraction_numerical': 0.031287605294825514, 'mean_word_length': 5.656, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '393438', 'license': 'Apache-2.0', 'n_tokens_mistral': 326, 'n_tokens_neox': 307, 'n_words': 64, 'path': 'notebooks/inspect_a_model.ipynb', 'repo_name': 'catskillsresearch/openasr20', 'size': '831'}