<jupyter_start><jupyter_text>Look for large Files<jupyter_code>import numpy as np
import zipfile, glob
from PIL import Image
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt
df = pd.DataFrame(columns=["pid","size"])
for zip_name in tqdm(sorted(glob.glob('data/*.zip'))):
    with zipfile.ZipFile(zip_name, "r") as zip_data:
        sizes = []
        names = []
        content_list = zip_data.namelist()
        for name_file in content_list:
            if name_file.endswith('.png'):
                img_bytes = zip_data.open(name_file)          # 1
                img_data = Image.open(img_bytes)
                sizes.append(img_data.size[0])
                names.append(zip_name)
        df = df.append(pd.DataFrame(data={"pid":names, "size":sizes}),ignore_index=True)
df['dateTime'] = pd.to_datetime(df['pid'].apply(lambda x: x.split("/")[1].split("_")[0][1:]))
fig, ax = plt.subplots()
df.groupby("dateTime")['size'].max().plot(marker='.',lw=0,ax=ax)
ax.set_xlim("20210717","20210722")
ax.set_ylabel("Max Hieght per Syringe(x*y)")
ax.set_yscale('log')
fname = "/opt/ifcb-data/power-buoy-deployment/D20210723T073937_IFCB161.roi"
files = ifcb.open_raw(fname) # creates an FilesetBin Object, this links each of the files
files.to_zip("../data/D20210723T073937_IFCB161.zip")
z = ifcb.open_zip("../data/D20210723T073937_IFCB161.zip")
# directory = ifcb.DataDirectory('/opt/ifcb-data/power-buoy-deployment/2021')
directory.
files.headers
files.to_zip("{}.zip".format(os.path.basename(os.path.splitext(fname)[0]))) # This will create a zip of the .pngs from the ROI files
# You can open the zip get the filenames or even open them
with zipfile.ZipFile("../data/D20210723T073937_IFCB161.zip", "r") as zip_data:
        content_list = zip_data.namelist()
        print(content_list)
        img_bytes = zip_data.open("D20210723T073937_IFCB161.csv")          # 1
#         img_data = Image.open(img_bytes)
import pandas as pd
pd.read_csv(img_bytes)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14777833250124814, 'fraction_numerical': 0.056415376934598103, 'mean_word_length': 3.899755501222494, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '507745', 'license': 'CC0-1.0', 'n_tokens_mistral': 808, 'n_tokens_neox': 733, 'n_words': 135, 'path': 'notebooks/ifcb-raw-view-images.ipynb', 'repo_name': 'patcdaniel/ifcb-scripts', 'size': '2003'}