<jupyter_start><jupyter_text>DLProfile Example using Imagenet Application  Set imports and neccessary environment variables<jupyter_code>import pathlib
import os
import sys
import matplotlib.pyplot as plt
import warnings
import pprint
import pandas as pd
import tensorflow as tf
import ntpath
import numpy as np
import math
VANIDL_DIR="{}".format(pathlib.Path(os.getcwd()).parent.parent.parent.absolute())
sys.path.insert(0, VANIDL_DIR)
warnings.filterwarnings('ignore')
os.environ["DARSHAN_DIR"] = "/soft/perftools/darshan/darshan-3.1.8"
os.environ["VANIDL_DIR"] = VANIDL_DIR<jupyter_output><empty_output><jupyter_text>Formatting<jupyter_code>pp = pprint.PrettyPrinter(indent=1)
class color:
    PURPLE = '\033[95m'
    CYAN = '\033[96m'
    DARKCYAN = '\033[36m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
    END = '\033[0m'<jupyter_output><empty_output><jupyter_text>Create instrance of DL Profile and load the darshan file<jupyter_code>from src.vanidl import VaniDL
profile = VaniDL()
#import shutil
#shutil.rmtree('/tmp/temp_analysis')
DATAPATH_INCLUDES = []
status = profile.Load("/home/dhari/darshan-logs/benchmark/imagenet/imagenet_run6_p8.darshan", data_paths_include=DATAPATH_INCLUDES)
if status:
    print("Darshan Trace loaded Successfully!")
else:
    print("Darshan Trace load Failed!")
    print(profile._error_str())<jupyter_output>[============================================================] 100.0% 264613 of 264613 Parsing DXT File 
[============================================================] 100.0% 22275 of 22275 Parsing Darshan File 
Darshan Trace loaded Successfully!<jupyter_text>Use Profile object to analyze the darshan I/O trace.  Verify if object worksThe GetDXTAsDF() function enables users to perform analysis<jupyter_code>pd.option_context('display.max_seq_items', None)
pd.set_option('display.max_rows', 512)
pd.set_option('display.max_colwidth', -1)
import numpy
numpy.set_printoptions(threshold=sys.maxsize)
df = profile.GetDXTAsDF()
print(df['Filename'].unique().to_list())
pp.pprint("Number of Files used in the application: {} files".format(df['Filename'].unique()))
pd.set_option('display.max_colwidth', -1)
df_normal = profile.GetTraceAsDF()
pp.pprint("Number of Files used in the application: {} files".format(df_normal['Filename'].unique()))<jupyter_output>['/dev/urandom', '/projects/datascience/dhari/dlio_datasets/imagenet/img_55_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_114_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_946_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_812_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_32_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_849_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_518_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_409_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_26_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_966_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_257_of_1024.tfrecords', '/projects/datascience/dhari/dlio_datasets/imagenet/img_295_of_1024.tfrecords', '/projects/datascience/dha[...]<jupyter_text>Collect the summary of the Application<jupyter_code>summary = profile.GetSummary()
print("\n")
print(color.BOLD + "Data Access Summary (from Darshan):"+ color.END)
print("Total Job time\t\t\t:\t{:0.2f} seconds".format(summary['job_time']))
#FIXME: calculate time per rank and then take max across it.
print("Time spent in I/O\t\t:\t{:0.2f} seconds".format(summary['total_io_time']))
print("% Time spent in I/O\t\t:\t{:0.2f}%".format(float(summary['total_io_time'])*100/summary['job_time']))
print("Total Data Accessed\t\t:\t{:0.2f} GB".format(float(summary['total_io_bytes'])/1024.0/1024.0/1024.0))
print("Data Access Modules used\t:\t{}".format(summary['io_interface_used']))
print("Data Operations\t\t\t:\t{}".format(summary['io_operations_used']))
print("# of files used\t\t\t:\t{}".format(len(summary['files_used'])))
print("# of MPI Ranks\t\t\t:\t{:0.0f} ranks".format(summary['num_ranks']))
      
print(color.UNDERLINE + "Data Transfer size:"+ color.END)
print("\tMin,Max\t\t\t:\t{:0.0f} bytes and {:0.0f} bytes".format(summary['data_transfer_size']['min'],summary['data_transfer_size']['max']))
print("\tAverage\t\t\t:\t{:0.0f} bytes".format(summary['data_transfer_size']['mean']))
print("\tMedian\t\t\t:\t{:0.0f} bytes".format(summary['data_transfer_size']['median']))
      
print(color.UNDERLINE + "Data Transfer bandwidth: (per rank)"+ color.END)
print("\tMin,Max\t\t\t:\t{:0.0f} B/s and {:0.0f} MB/s".format(summary['data_transfer_bandwidth']['min'],summary['data_transfer_bandwidth']['max']/1024.0/1024.0))
print("\tAverage\t\t\t:\t{:0.0f} MB/s".format(summary['data_transfer_bandwidth']['mean']/1024.0/1024.0))
print("\tMedian\t\t\t:\t{:0.0f} MB/s".format(summary['data_transfer_bandwidth']['median']/1024.0/1024.0))
      
print(color.UNDERLINE + "Access Pattern:"+ color.END)
print("\tSequential\t\t:\t{:0.2f}%".format(float(summary['access_pattern']['sequential'])))
print("\tConsecutive\t\t:\t{:0.2f}%".format(float(summary['access_pattern']['consecutive'])))
#An I/O op issued at an offset greater than where the previous I/O op ended.
#An I/O op issued at the offset immediately after the end of the previous I/O

print("\n")
print(color.BOLD + "Files Summary:"+ color.END)
print("File Types\t\t\t:\t{}".format("tfrecord"))
print(color.UNDERLINE + "Dataset Size:"+ color.END)
print("\tTotal\t\t\t:\t{:0.3f} GB".format(float(summary['file_used_summary']['size']['total'])/1024.0/1024.0/1024.0))
print("\tMin,Max\t\t\t:\t{:0.3f} GB and {:0.3f} GB".format(float(summary['file_used_summary']['size']['min'])/1024.0/1024.0/1024.0,float(summary['file_used_summary']['size']['max'])/1024.0/1024.0/1024.0))
print("\tAverage\t\t\t:\t{:0.3f} GB".format(float(summary['file_used_summary']['size']['mean'])/1024.0/1024.0/1024.0))
pp.pprint("Job time : {} seconds".format(profile.GetJobTime()))
pp.pprint("Time spent by application on I/O: {} seconds".format(profile.GetIOTime()))<jupyter_output>'Time spent by application on I/O: 18.116012500073694 seconds'<jupyter_text>I/O time spent on each file<jupyter_code>filename_series = []
time_series = []
for file in df['Filename'].unique():
    filename_series.append(os.path.splitext(ntpath.basename(file))[0][6:11])
    time_series.append(profile.GetIOTime(filepath=file))

zipped_lists = zip(filename_series, time_series)
sorted_pairs = sorted(zipped_lists)
tuples = zip(*sorted_pairs)
filename_series, time_series = [ list(tuple) for tuple in  tuples]
points_x = 30
for i in range(math.ceil(float(len(time_series))/points_x)):
    start = i * points_x
    end = (i+1) * points_x
    plt.figure(figsize=(20,4))
    plt.grid()
    plt.plot(filename_series[start:end], time_series[start:end]);
    plt.show()<jupyter_output><empty_output><jupyter_text>I/O Time spent per rank<jupyter_code>for rank in df['Rank'].unique():
    print("I/O time for rank {}: {:0.2f} seconds".format(rank,profile.GetIOTime(rank=rank)))
"Total I/O performed by application: {:0.2f} GB".format(float(profile.GetIOSize())/1024.0/1024.0/1024.0)<jupyter_output><empty_output><jupyter_text>I/O performed on each file<jupyter_code>filename_series = []
size_series = []
for file in df['Filename'].unique():
    filename_series.append(os.path.splitext(ntpath.basename(file))[0][6:11])
    size_series.append(float(profile.GetIOSize(filepath=file))/1024.0/1024.0)

zipped_lists = zip(filename_series, size_series)
sorted_pairs = sorted(zipped_lists)
tuples = zip(*sorted_pairs)
filename_series, size_series = [ list(tuple) for tuple in  tuples]



plt.figure(figsize=(20,4))
size_series = np.array(size_series)
plt.hist(size_series,bins = 100)
plt.xlabel("I/O Size per file")
plt.ylabel("File counts")
plt.show()
    
# points_x = 30
# for i in range(math.ceil(float(len(size_series))/points_x)):
#     start = i * points_x
#     end = (i+1) * points_x
#     plt.figure(figsize=(20,4))
#     plt.grid()
#     plt.plot(filename_series[start:end], size_series[start:end]);
#     plt.show()
for rank in df['Rank'].unique():
    print("I/O performed by rank {}: {:0.2f} MB".format(rank, float(profile.GetIOSize(rank=rank))/1024.0/1024.0))
print("Average Size of dataset {:0.2f} MB".format(np.mean(list(profile.GetFileSizes().values()))/1024.0/1024.0))<jupyter_output>Average Size of dataset 254.02 MB<jupyter_text>How application access data over time.<jupyter_code>tl = profile.CreateIOTimeline(is_print=False)
plt.figure(figsize=(8,4))
plt.xlabel("Timeline (ms)")
plt.ylabel("# of I/O Operations")
plt.grid()
plt.plot(tl['time_step'], tl['operation_count']);<jupyter_output>[============================================================] 100.0% 262936 of 262936 Creating Timeline<jupyter_text>How files are accessed over the duration of the Job.<jupyter_code>filename_series = []
operation_series = []
for file in df['Filename'].unique()[:8]:
    tl = profile.CreateIOTimeline(filepath=file,time_step=0.001,is_print=False,save=False)
    filename_series.append(os.path.splitext(ntpath.basename(file))[0][6:11])
    operation_series.append(tl['operation_count'].sum())

plt.figure(figsize=(20,4))
operation_series = np.array(operation_series)
plt.hist(operation_series,bins = 100);
plt.show()<jupyter_output>[============================================================] 100.0% 24 of 24 Creating Timeline 
[============================================================] 100.0% 1027 of 1027 Creating Timeline 
[============================================================] 100.0% 1027 of 1027 Creating Timeline 
[============================================================] 100.0% 1027 of 1027 Creating Timeline 
[============================================================] 100.0% 1027 of 1027 Creating Timeline 
[============================================================] 100.0% 1027 of 1027 Creating Timeline 
[============================================================] 100.0% 1027 of 1027 Creating Timeline 
[============================================================] 100.0% 1027 of 1027 Creating Timeline<jupyter_text>Show how each file is accessed by each rank.<jupyter_code>fig, axs = plt.subplots(3, 3, figsize=(20, 10))
for rank in df['Rank'].unique()[:8]:
    x = int(rank/3)
    y = rank%3
    tl = profile.CreateIOTimeline(rank=rank,save=False)
    tl.plot(x='time_step',y='operation_count', title=rank, ax=axs[x][y])
plt.show()<jupyter_output>[============================================================] 100.0% 32867 of 32867 Creating Timeline 
[============================================================] 100.0% 32867 of 32867 Creating Timeline 
[============================================================] 100.0% 32867 of 32867 Creating Timeline 
[============================================================] 100.0% 32867 of 32867 Creating Timeline 
[============================================================] 100.0% 32867 of 32867 Creating Timeline 
[============================================================] 100.0% 32867 of 32867 Creating Timeline 
[============================================================] 100.0% 32867 of 32867 Creating Timeline 
[============================================================] 100.0% 32867 of 32867 Creating Timeline<jupyter_text>Data Transfer Size distribution within the application<jupyter_code>request_df = profile.GetIORequestDistribution()
df['Length'].plot(kind='hist', figsize=(5, 3));
plt.xlabel("Transfer Size in bytes")<jupyter_output><empty_output><jupyter_text>Data Transfer Size distribution for each file.<jupyter_code>filename_series = []
request_mean = []
for file in df['Filename'].unique()[:8]:
    tl = profile.GetIORequestDistribution(filepath=file)
    request_mean.append(tl.mean())
plt.figure(figsize=(5,3))
request_mean = np.array(request_mean)
plt.hist(request_mean,bins = 10);
plt.ylabel("# files accessed")
plt.xlabel("# of requests per file")
plt.grid()
plt.show()<jupyter_output><empty_output><jupyter_text>Data Transfer Sizes per Rank<jupyter_code>fig, axs = plt.subplots(3, 3, figsize=(20, 10))
for rank in df['Rank'].unique()[:8]:
    x = int(rank/3)
    y = rank%3
    tl = profile.GetIORequestDistribution(rank=rank)
    tl.plot(kind='bar', title=rank, ax=axs[x][y])<jupyter_output><empty_output><jupyter_text>File summary of each file accessed by the Application<jupyter_code>pp = pprint.PrettyPrinter(indent=1)
count =1
for file in df['Filename'].unique()[:8]:
    if 'h5' in file:
        pp.pprint(profile.GetFileSummary(file,ext='h5'))
    else:
        if count == 1:
            count+=1
            pp.pprint(profile.GetFileSummary(file,ext='tfrecord',tf_record_features={
      'image/encoded': tf.io.FixedLenFeature((), dtype=tf.string, default_value=''),
      'image/source_id': tf.io.FixedLenFeature((), tf.string, default_value=''),
      'image/height': tf.io.FixedLenFeature((), tf.int64, default_value=1),
      'image/width': tf.io.FixedLenFeature((), tf.int64, default_value=1),
      'image/object/bbox/xmin': tf.io.VarLenFeature(dtype=tf.float32),
      'image/object/bbox/ymin': tf.io.VarLenFeature(dtype=tf.float32),
      'image/object/bbox/xmax': tf.io.VarLenFeature(dtype=tf.float32),
      'image/object/bbox/ymax': tf.io.VarLenFeature(dtype=tf.float32),
      'image/object/class/label': tf.io.VarLenFeature(dtype=tf.int64),
      'image/object/class/text': tf.io.FixedLenFeature((), dtype=tf.string, default_value=''),
      'image/colorspace': tf.io.FixedLenFeature((), tf.string, default_value=''),
      'image/channels': tf.io.FixedLenFeature((), tf.int64, default_value=1),
      'image/format': tf.io.FixedLenFeature((), tf.string, default_value=''),
      'image/filename': tf.io.FixedLenFeature((), tf.string, default_value='')
  }))<jupyter_output>{'ext': 'tfrecord',
 'filename': 'urandom',
 'io_size': 28488,
 'io_time': 0.001962500000000005,
 'path': '/dev/urandom',
 'size': 0,
 'special': <ParallelMapDataset shapes: {image/object/bbox/xmax: (None,), image/object/bbox/xmin: (None,), image/object/bbox/ymax: (None,), image/object/bbox/ymin: (None,), image/object/class/label: (None,), image/channels: (), image/colorspace: (), image/encoded: (), image/filename: (), image/format: (), image/height: (), image/object/class/text: (), image/source_id: (), image/width: ()}, types: {image/object/bbox/xmax: tf.float32, image/object/bbox/xmin: tf.float32, image/object/bbox/ymax: tf.float32, image/object/bbox/ymin: tf.float32, image/object/class/label: tf.int64, image/channels: tf.int64, image/colorspace: tf.string, image/encoded: tf.string, image/filename: tf.string, image/format: tf.string, image/height: tf.int64, image/object/class/text: tf.string, image/source_id: tf.string, image/width: tf.int64}>}
{'fraction_non_alphanumeric': 0.23511368940454758, 'fraction_numerical': 0.049743281989731276, 'mean_word_length': 6.529116465863454, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '537773', 'license': 'MIT', 'n_tokens_mistral': 5720, 'n_tokens_neox': 5146, 'n_words': 854, 'path': 'examples/benchmark/8/Imagenet.ipynb', 'repo_name': 'hariharan-devarajan/dlprofiler', 'size': '14997'}