<commit_before>\documentclass{beamer}

% You can also use a 16:9 aspect ratio:
%\documentclass[aspectratio=169]{beamer}
\usetheme{TACC16}
\usepackage{graphicx}

% It's possible to move the footer to the right:
%\usetheme[rightfooter]{TACC16}


% Detailed knowledge of application workload characteristics can
% optimize performance of current and future systems. This may sound
% daunting, with many HPC data centers hosting over 2,000 users running
% thousands of applications and millions of jobs per month.  XALT is an
% open source tool developed at the Texas Advanced Computing Center
% (TACC) that collects system usage information to quantitatively report
% how users are using your system. This session will explore the
% benefits of detailed application workload profiling and how the XALT
% tool has helped leading supercomputing sites unlock the power of their
% application usage data. 


\begin{document}
\title[XALT]{Extracting Useful Data from XALT DB}
\author{<NAME>} 
\date{Jan. 20, 2022} 

% page 1
\frame{\titlepage} 

% page 2
\begin{frame}{XALT: Outline}
  \center{\includegraphics[width=.9\textwidth]{XALT_Sticker.png}}
  \begin{itemize}
    \item XALT Parts: Generating and Storing
    \item Quick Discussion about what is stored
    \item Caveats about XALT Records
    \item xalt\_usage\_report.py
    \item Kinds of reports available
    \item What kinds of reports would sites like?
    \item Future Topics for XALT zoom mtg.
  \end{itemize}
\end{frame}

% page 3
\begin{frame}{XALT Generating and Storing}
  \begin{itemize}
    \item ALL programs on system have XALT via LD\_PRELOAD
    \item Some prgms generate *.json records
    \item These records can be written to a MySQL DB (not required) 
    \item Afterward *.json records are deleted.
  \end{itemize}
\end{frame}

% page 4
\begin{frame}{Caveats about XALT records}
  \begin{itemize}
    \item Core-Hours not Node-Hours 
    \item XALT knows about mpi tasks and threads
    \item It doesn't know if a node is shared.
    \item User might run multiple single core prgms on a node
    \item User might run ingle core prgms on one or more nodes
    \item XALT runs inside each program.
    \item It is blind to what happens outside of user prgm.
    \item It would require a daemon on each node to know.
  \end{itemize}
\end{frame}

% page 5
\begin{frame}{XALT time records won't match Accounting}
  \begin{itemize}
    \item XALT will filter out or sample programs
    \item It won't catch them all.
    \item XALT can double count (rarely)
    \item If user forks off another prgm $\Rightarrow$ double counting
    \item If user runs $>$ 1 prgm per core.
    \item XALT uses real time not cpu time.
  \end{itemize}
\end{frame}

% page 6
\begin{frame}{Python prgm: sbin/xalt\_usage\_report.py}
  \begin{itemize}
    \item An attempt to be system agnostic.
    \item It provides a name mapping file.
    \item It reports data that TACC has found useful:
      \begin{itemize}
        \item Overall Job counts
        \item Self Built vs. Not.
        \item It reports Top Execs by Core-Hours, Number of Runs,
          Number of users for All, MPI Only, Scalar
        \item Top Module usage
        \item Compiler usage
        \item Library Usage
      \end{itemize}
  \end{itemize}
\end{frame}

% page 7
\begin{frame}{Overall Job counts}
  \begin{itemize}
    \item Core hours for ``system'' prgms vs. user prgms.
    \item System prgms come from a module, User prgms don't.
    \item We see about 5\% for system prgms.
    \item Still have to teach how to build program.
  \end{itemize}
\end{frame}

% page 8
\begin{frame}{Self-Built vs. Not.}
  \begin{itemize}
    \item Beside system supplied program we have groups that share prgms
    \item Like to track that.
    \item Prgms built under XALT know build user.
    \item XALT knows the run user.
    \item We report 2 to 4 \% non self-built prgm runs.
  \end{itemize}
\end{frame}

% page 9
\begin{frame}[fragile]
    \frametitle{xalt\_name\_mapping.py}
 {\small
    \begin{semiverbatim}
equiv_patternA = [
    [ r'^pmemd'               , 'Amber*'      ],
    [ r'^sander'              , 'Amber*'      ],
    [ r'^absorption\.real\.x' , 'BerkeleyGW*' ],
    [ r'^absorption\.cplx\.x' , 'BerkeleyGW*' ],
    [ r'^kernel\.real\.x'     , 'BerkeleyGW*' ],
    \end{semiverbatim}
}
  \begin{itemize}
    \item Map prgms to projects.
    \item Sites may need to modify to match their site.
  \end{itemize}

\end{frame}


% page 13
\begin{frame}{Future Topics?} 
  \begin{itemize}
    \item Package tracking
    \item Extracting results from the DB.
    \item Others?
  \end{itemize}
\end{frame}
%

%\input{./themes/license}

\end{document}<commit_msg>Updating 1/20 presentation<commit_after>\documentclass{beamer}

% You can also use a 16:9 aspect ratio:
%\documentclass[aspectratio=169]{beamer}
\usetheme{TACC16}
\usepackage{graphicx}

% It's possible to move the footer to the right:
%\usetheme[rightfooter]{TACC16}


% Detailed knowledge of application workload characteristics can
% optimize performance of current and future systems. This may sound
% daunting, with many HPC data centers hosting over 2,000 users running
% thousands of applications and millions of jobs per month.  XALT is an
% open source tool developed at the Texas Advanced Computing Center
% (TACC) that collects system usage information to quantitatively report
% how users are using your system. This session will explore the
% benefits of detailed application workload profiling and how the XALT
% tool has helped leading supercomputing sites unlock the power of their
% application usage data. 


\begin{document}
\title[XALT]{Extracting Useful Data from XALT DB}
\author{<NAME>} 
\date{Jan. 20, 2022} 

% page 1
\frame{\titlepage} 

% page 2
\begin{frame}{XALT: Outline}
  \center{\includegraphics[width=.9\textwidth]{XALT_Sticker.png}}
  \begin{itemize}
    \item XALT Parts: Generating and Storing
    \item Quick Discussion about what is stored
    \item Caveats about XALT Records
    \item xalt\_usage\_report.py
    \item Kinds of reports available
    \item What kinds of reports would sites like?
    \item Future Topics for XALT zoom mtg.
  \end{itemize}
\end{frame}

% page 3
\begin{frame}{XALT Generating and Storing}
  \begin{itemize}
    \item ALL programs on system have XALT via LD\_PRELOAD
    \item Some prgms generate *.json records
    \item These records can be written to a MySQL DB (not required) 
    \item Afterward *.json records are deleted.
  \end{itemize}
\end{frame}

% page 4
\begin{frame}{Caveats about XALT records}
  \begin{itemize}
    \item Core-Hours not Node-Hours 
    \item XALT knows about mpi tasks and threads
    \item It doesn't know if a node is shared.
    \item User might run multiple single core prgms on a node
    \item User might run ingle core prgms on one or more nodes
    \item XALT runs inside each program.
    \item It is blind to what happens outside of user prgm.
    \item It would require a daemon on each node to know.
  \end{itemize}
\end{frame}

% page 5
\begin{frame}{XALT time records won't match Accounting}
  \begin{itemize}
    \item XALT will filter out or sample programs
    \item It won't catch them all.
    \item XALT can double count (rarely)
    \item If user forks off another prgm $\Rightarrow$ double counting
    \item If user runs $>$ 1 prgm per core.
    \item XALT uses real time not cpu time.
  \end{itemize}
\end{frame}

% page 6
\begin{frame}{Python prgm: sbin/xalt\_usage\_report.py}
  \begin{itemize}
    \item An attempt to be system agnostic.
    \item It provides a name mapping file.
    \item It reports data that TACC has found useful:
      \begin{itemize}
        \item Overall Job counts
        \item Self Built vs. Not.
        \item It reports Top Execs by Core-Hours, Number of Runs,
          Number of users for All, MPI Only, Scalar
        \item Top Module usage
        \item Compiler usage
        \item Library Usage
      \end{itemize}
  \end{itemize}
\end{frame}

% page 7
\begin{frame}{Overall Job counts}
  \begin{itemize}
    \item Core hours for ``system'' prgms vs. user prgms.
    \item System prgms come from a module, User prgms don't.
    \item We see about 5\% for system prgms.
    \item Still have to teach how to build program.
  \end{itemize}
\end{frame}

% page 8
\begin{frame}{Self-Built vs. Not.}
  \begin{itemize}
    \item Beside system supplied program we have groups that share prgms
    \item Like to track that.
    \item Prgms built under XALT know build user.
    \item XALT knows the run user.
    \item We report 2 to 4 \% non self-built prgm runs.
  \end{itemize}
\end{frame}

% page 9
\begin{frame}[fragile]
    \frametitle{xalt\_name\_mapping.py}
 {\small
    \begin{semiverbatim}
equiv_patternA = [
    [ r'^pmemd'               , 'Amber*'      ],
    [ r'^sander'              , 'Amber*'      ],
    [ r'^absorption\.real\.x' , 'BerkeleyGW*' ],
    [ r'^absorption\.cplx\.x' , 'BerkeleyGW*' ],
    [ r'^kernel\.real\.x'     , 'BerkeleyGW*' ],
    \end{semiverbatim}
}
  \begin{itemize}
    \item Map prgms to projects
    \item Sites may need to modify to match their site
    \item Names get a \* when mapped
    \item Nothing stops a user naming Hello $\Rightarrow$ pmemd
  \end{itemize}

\end{frame}

% page 10
\begin{frame}[fragile]
    \frametitle{Typical Reports: Top Execs Core-Hours}
 {\small
    \begin{semiverbatim}
    CoreHrs   # Runs  # Users  # Accts  Exec
    -------   ------  -------  -------  ----
289,924,113      710       13        7  CESM*
 68,162,422  174,594        8        4  Chroma*
 51,135,767  102,957        5        3  gene_frontera
 39,173,897   22,666       44       32  LAMMPS*
 38,714,742   75,751       50       39  NAMD*
 35,858,260  254,470       55       37  VASP*
    \end{semiverbatim}
}
\end{frame}

% page 11
\begin{frame}[fragile]
    \frametitle{Typical Reports: Top Execs Runs}
 {\small
    \begin{semiverbatim}
   CoreHrs   # Runs  # Users  # Accts  Exec
   -------   ------  -------  -------  ----
 2,391,403  815,586        1        2  Rosetta*
     2,402  583,091      136       97  mv
     7,068  515,852      373      211  grep
11,942,647  499,463      391      218  Python*
    24,176  414,788        1        2  MOPAC2016.exe
 2,852,787  303,295        4        2  rockstar-galaxies
    \end{semiverbatim}
}
\end{frame}

% page 12
\begin{frame}[fragile]
    \frametitle{Typical Reports: Top Execs Users}
 {\small
    \begin{semiverbatim}
   CoreHrs   # Runs  # Users  # Accts  Exec
   -------   ------  -------  -------  ----
11,942,647  499,463      391      218  Python*
     7,068  515,852      373      211  grep
     1,330   67,595      335      197  sed
    12,779  121,983      322      182  gawk
     2,402  583,091      136       97  mv
    16,366  131,215      132       94  cp
   137,580   21,731      103       62  perl
    \end{semiverbatim}
}
\end{frame}


% page 11
\begin{frame}{Side Note: Integrating XALT testing into github}
  \begin{itemize}
    \item I would like to use github actions
    \item Have not figured out how to setup mysql inside container
  \end{itemize}
\end{frame}


% page 12
\begin{frame}{Conclusions}
  \begin{itemize}
    \item xalt\_extract\_record user\_program to see the XALT
      Watermark
    \item Use file transport to check *.json
    \item Use logger transport to check syslog tracking
    \item Possibly use xalt testing locally.
    \item Willing to work with anyone who tries any system.
  \end{itemize}
\end{frame}


% page 13
\begin{frame}{Future Topics?} 
  \begin{itemize}
    \item Package tracking
    \item Extracting results from the DB.
    \item Others?
  \end{itemize}
\end{frame}
%

%\input{./themes/license}

\end{document}
{'fraction_non_alphanumeric': 0.09900905518537502, 'fraction_numerical': 0.033914232017768664, 'mean_word_length': 2.639104755983836, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4329092', 'n_tokens_mistral': 4180, 'n_tokens_neox': 3751, 'n_words': 1390}