<jupyter_start><jupyter_text>Collectl Mem Log Analysis  Functionalities- Plot memory utilization graphs. InputLog files are read from a directory in `../data`. This directory is assumed to have the following structure:```logs/  [node-1]/    collectl.tar.gz  ...  [node-n]/    collectl.tar.gz```A tarball `collectl.tar.gz` contains log files. The log file extension identifies the type of resource monitored:- `.cpu.gz`: CPU monitoring log file.- `.numa.gz`: memory monitoring log file.- `.dsk.gz`: disk I/O monitoring log file.  Notebook Configuration<jupyter_code>########## GENERAL
# Name of the directory in `../data`
EXPERIMENT_DIRNAME = "BuzzBlogBenchmark_2022-01-15-18-52-05"
# Ramp up duration (in sec)
RAMP_UP_DURATION = 720
# Ramp down duration (in sec)
RAMP_DOWN_DURATION = 360

########## MEMORY
# Analyzed metric (options: "used", "free", "slab", "mapped", "anon", "anonh", "inactive", "hits")
COLLECTL_MEM_METRIC = "free"
# Filter Mem nodes
COLLECTL_MEM_NODES = None<jupyter_output><empty_output><jupyter_text>Notebook Setup<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import sys
import warnings
warnings.filterwarnings("ignore")

sys.path.append(os.path.abspath(os.path.join("..")))
from parsers.collectl_parser import CollectlParser
from utils.utils import *<jupyter_output><empty_output><jupyter_text>Log Parsing<jupyter_code># Get experiment time range, excluding ramping periods
experiment_start_time = get_experiment_start_time(EXPERIMENT_DIRNAME)
start_time = experiment_start_time + np.timedelta64(RAMP_UP_DURATION, 's')
experiment_end_time = get_experiment_end_time(EXPERIMENT_DIRNAME)
end_time = experiment_end_time - np.timedelta64(RAMP_DOWN_DURATION, 's')
# Build data frames
mem = pd.concat([
    pd.DataFrame.from_dict(CollectlParser(logfile, "mem", start_time, end_time).parse()).assign(node_name=node_name)
    for node_name, logfile in get_collectl_mem_logfiles(EXPERIMENT_DIRNAME)
], ignore_index=True)
# Filter data frames
mem = mem[(mem["hw_metric"] == COLLECTL_MEM_METRIC)]
if COLLECTL_MEM_NODES:
    mem = mem[(mem["hw_no"].isin(COLLECTL_MEM_NODES))]
mem["hw_no"].unique()
# (Re) Build columns
mem["timestamp"] = mem.apply(lambda r: (r["timestamp"] - start_time).total_seconds(), axis=1)
mem["window"] = mem.apply(lambda r: int(r["timestamp"]), axis=1)
mem["value"] = mem.apply(lambda r: float(r["value"]), axis=1)
# (Re) Create index
mem.set_index("timestamp", inplace=True)
mem.sort_index(inplace=True)
# Get values
node_names = get_node_names(EXPERIMENT_DIRNAME)<jupyter_output><empty_output><jupyter_text>Memory Monitoring<jupyter_code># Plot Mem utilization (1-sec granularity)
fig = plt.figure(figsize=(24, len(node_names) * 12))
for (i, node_name) in enumerate(node_names):
    df = mem[(mem["node_name"] == node_name)]
    df = df.groupby(["window"])["value"].mean()
    ax = fig.add_subplot(len(node_names), 1, i + 1)
    ax.set_xlim((df.index.min(), df.index.max()))
    ax.grid(alpha=0.75)
    df.plot(ax=ax, kind="line", title="%s - Mem Utilization" % node_name, xlabel="Time (seconds)",
            ylabel="%s" % COLLECTL_MEM_METRIC, grid=True)
########## LOCAL CONFIG
# Minimum time (in seconds)
MIN_TIME = None
# Maximum time (in seconds)
MAX_TIME = None

# Plot Mem utilization (millisec granularity)
if MIN_TIME and MAX_TIME:
    fig = plt.figure(figsize=(24, len(node_names) * 12))
    for (i, node_name) in enumerate(node_names):
        df = mem[(mem["node_name"] == node_name)]
        df = df[(df.index >= MIN_TIME) & (df.index <= MAX_TIME)]
        df = df.groupby(["timestamp", "hw_no"])["value"].mean()
        df = df.unstack()
        ax = fig.add_subplot(len(node_names), 1, i + 1)
        ax.set_xlim((df.index.min(), df.index.max()))
        ax.grid(alpha=0.75)
        df.plot(ax=ax, kind="line", title="%s - Mem Utilization" % node_name, xlabel="Time (seconds)",
                ylabel="%s" % COLLECTL_MEM_METRIC, grid=True, legend=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1508868348738446, 'fraction_numerical': 0.011741194104421683, 'mean_word_length': 5.355555555555555, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '112995', 'license': 'Apache-2.0', 'n_tokens_mistral': 1469, 'n_tokens_neox': 1418, 'n_words': 315, 'path': 'analysis/notebooks/CollectlMemLogAnalysis.ipynb', 'repo_name': 'Bwei112233/BuzzBlogBenchmark', 'size': '4003'}