<commit_before>%!TEX root = hard_stack_paper/paper.tex

\begin{figure*}[t]
\begin{subfigure}[t]{\textwidth}
\centering
\scalebox{0.72}{%
\begin{tikzpicture}
    \def\dx{21pt}
    \def\dy{11pt}
    \def\sy{12*\dy}
    \def\oxb{8*\dx}
    \def\by{0pt}
    \def\ox{0*\oxb}

    \tikzstyle{label}=[text width=35mm,align=center,text height=2mm]    
    \tikzstyle{word}=[text width=35mm,align=center,text height=2mm]    
    \tikzstyle{tracker}=[fill=red!40,text width=15mm,align=center,text height=2mm]
    \tikzstyle{softmax}=[fill=blue!40,text width=15mm,align=center,text height=2mm]
    \tikzstyle{comp}=[fill=green!40,text width=20mm,align=center,text height=2mm]
    \tikzstyle{compoff}=[fill=green!10!black!10,text width=20mm,align=center,text height=2mm]
    \tikzstyle{result}=[line width=1pt,draw=black,text width=15mm,align=center,text height=2mm]    
    \tikzstyle{sbox}=[line width=1pt,draw=black,text width=25mm,align=center,text height=13.3mm]
    \tikzstyle{bbox}=[line width=1pt,draw=black,text width=25mm,align=center,text height=13.3mm]
    \tikzstyle{focus1}=[fill=yellow!40,text width=25mm,align=center,text height=2mm]
    \tikzstyle{focus2}=[fill=yellow!40,text width=25mm,align=center,text height=5.5mm]

    \node[label]  (sl) at (\ox-0.35*\oxb+0*\dx,\by+1*\dy) {buffer};

    \node[focus1] (0bb) at  (\ox+0*\dx,2.5*\dy) {};
    \node[word]  (0b3) at (\ox+0*\dx,\by-0.5*\dy) {};
    \node[word]  (0b2) at (\ox+0*\dx,\by+1.5*\dy) {down};
    \node[word]  (0b1) at (\ox+0*\dx,\by+2.5*\dy) {sat};
    \node[bbox] (0bb) at  (\ox+0*\dx,\by+1.0*\dy) {};
    
    \node[label]  (sl) at (\ox-0.35*\oxb+0*\dx,\sy+0.5*\dy) {stack};
    
    \node[focus2] (0sb) at  (\ox+0*\dx,\sy-0.5*\dy) {};
    \node[word]  (0s1) at (\ox+0*\dx,\sy-1*\dy) {cat};
    \node[word]  (0s2) at (\ox+0*\dx,\sy+0*\dy) {the};
    \node[word]  (0s3) at (\ox+0*\dx,\sy+1*\dy) {};
    \node[sbox] (0sb) at  (\ox+0*\dx,\sy+0.5*\dy) {};
    
    \node[comp] (0c) at  (\ox+0.5*\oxb,\sy-1.5*\dy) {composition};
    
    \node[tracker] (0t) at  (\ox+0*\dx,5*\dy) {tracking};
    \node[softmax] (0sm) at  (\ox+3.25*\dx,6.25*\dy) {transition};
    \node[result] (0so) at  (\ox+3.25*\dx,8.5*\dy) {\reduce};
    
    \def\ox{1*\oxb}

    \node[focus1] (1bb) at  (\ox+0*\dx,2.5*\dy) {};
    \node[word]  (1b3) at (\ox+0*\dx,\by-0.5*\dy) {};
    \node[word]  (1b2) at (\ox+0*\dx,\by+1.5*\dy) {down};
    \node[word]  (1b1) at (\ox+0*\dx,\by+2.5*\dy) {sat};
    \node[bbox] (1bb) at  (\ox+0*\dx,\by+1*\dy) {};
    
    \node[focus2] (1sb) at  (\ox+0*\dx,\sy-0.5*\dy) {};
    \node[word]  (1s1) at (\ox+0*\dx,\sy-1*\dy) {the cat};
    \node[word]  (1s2) at (\ox+0*\dx,\sy+0*\dy) {};
    \node[word]  (1s3) at (\ox+0*\dx,\sy+1*\dy) {};
    \node[sbox] (1sb) at  (\ox+0*\dx,\sy+0.5*\dy) {};
    
    \node[compoff] (1c) at  (\ox+0.5*\oxb,\sy-1.5*\dy) {composition};
    
    \node[tracker] (1t) at  (\ox+0*\dx,5*\dy) {tracking};
    \node[softmax] (1sm) at  (\ox+3.25*\dx,6.25*\dy) {transition};
    \node[result] (1so) at  (\ox+3.25*\dx,8.5*\dy) {\shift};
     
    \def\ox{2*\oxb}

    \node[focus1] (2bb) at  (\ox+0*\dx,2.5*\dy) {};
    \node[word]  (2b3) at (\ox+0*\dx,\by-0.5*\dy) {};
    \node[word]  (2b2) at (\ox+0*\dx,\by+1.5*\dy) {};
    \node[word]  (2b1) at (\ox+0*\dx,\by+2.5*\dy) {down};
    \node[bbox] (2bb) at  (\ox+0*\dx,\by+1*\dy) {};
    
    \node[focus2] (2sb) at  (\ox+0*\dx,\sy-0.5*\dy) {};
    \node[word]  (2s1) at (\ox+0*\dx,\sy-1*\dy) {sat};
    \node[word]  (2s2) at (\ox+0*\dx,\sy+0*\dy) {the cat};
    \node[word]  (2s3) at (\ox+0*\dx,\sy+1*\dy) {};
    \node[sbox] (2sb) at  (\ox+0*\dx,\sy+0.5*\dy) {};
   
    \node[tracker] (2t) at  (\ox+0*\dx,5*\dy) {tracking};

    
    \pgfsetarrowsend{latex}
    \tikzstyle{fwd} = [draw=black, line width=1pt]
    \tikzstyle{gated} = [draw=black!50, line width=0.8pt]

    \draw [fwd] (0sb) -- (0t);
    \draw [fwd] (0bb) -- (0t);
    \draw [fwd] (0t) -- (0sm);
    \draw [fwd] (0sm) -- (0so);
    \draw [fwd] (0sb) -- (0c);
    
    \draw [fwd] (0t) -- (1t);
    \draw [fwd] (0t) to[out=50,in=-175] (0c);
    \draw [fwd] (0sb) -- (1sb);
    \draw [fwd] (0bb) -- (1bb);
    \draw [fwd] (0so) to[out=5,in=-140] (1sb);
    \draw [fwd] (0so) to[out=-5,in=170] (1bb);
    \draw [gated] (0bb) to[out=15,in=-125] (1sb);
    \draw [fwd] (0c) -- (1sb);

    \draw [fwd] (1sb) -- (1t);
    \draw [fwd] (1bb) -- (1t);
    \draw [fwd] (1t) -- (1sm);
    \draw [fwd] (1sm) -- (1so);
    \draw [fwd] (1sb) -- (1c);
    
    \draw [fwd] (1t) -- (2t);
    \draw [fwd] (1t) to[out=50,in=-175] (1c);
    \draw [fwd] (1sb) -- (2sb);
    \draw [fwd] (1bb) -- (2bb);
    \draw [fwd] (1so) to[out=5,in=-140] (2sb);
    \draw [fwd] (1so) to[out=-5,in=170] (2bb);
    \draw [fwd] (1bb) to[out=15,in=-125] (2sb);
    \draw [gated] (1c) -- (2sb);

    \draw [fwd] (2sb) -- (2t);
    \draw [fwd] (2bb) -- (2t);


  \end{tikzpicture}}
  
 \caption{The SPINN model unrolled for two transitions during the processing of the sentence \word{the cat sat down}. `Tracking', `transition', and `composition' are neural network layers. Gray arrows indicate connections which are blocked by a gating function.}\label{fig:model:1d}
  
\end{subfigure}\\\\\\
\begin{subfigure}[t]{\textwidth}
\centering
\scalebox{0.49}{%
\begin{tikzpicture}
    \def\dx{20pt}
    \def\dy{11pt}
    \def\sy{7*\dy}
    \def\oxb{5.5*\dx}<commit_msg>Size tweak.<commit_after>%!TEX root = hard_stack_paper/paper.tex

\begin{figure*}[t]
\begin{subfigure}[t]{\textwidth}
\centering
\scalebox{0.72}{%
\begin{tikzpicture}
    \def\dx{21pt}
    \def\dy{11pt}
    \def\sy{12*\dy}
    \def\oxb{8*\dx}
    \def\by{0pt}
    \def\ox{0*\oxb}

    \tikzstyle{label}=[text width=35mm,align=center,text height=2mm]    
    \tikzstyle{word}=[text width=35mm,align=center,text height=2mm]    
    \tikzstyle{tracker}=[fill=red!40,text width=15mm,align=center,text height=2mm]
    \tikzstyle{softmax}=[fill=blue!40,text width=15mm,align=center,text height=2mm]
    \tikzstyle{comp}=[fill=green!40,text width=20mm,align=center,text height=2mm]
    \tikzstyle{compoff}=[fill=green!10!black!10,text width=20mm,align=center,text height=2mm]
    \tikzstyle{result}=[line width=1pt,draw=black,text width=15mm,align=center,text height=2mm]    
    \tikzstyle{sbox}=[line width=1pt,draw=black,text width=25mm,align=center,text height=13.3mm]
    \tikzstyle{bbox}=[line width=1pt,draw=black,text width=25mm,align=center,text height=13.3mm]
    \tikzstyle{focus1}=[fill=yellow!40,text width=25mm,align=center,text height=2mm]
    \tikzstyle{focus2}=[fill=yellow!40,text width=25mm,align=center,text height=5.5mm]

    \node[label]  (sl) at (\ox-0.35*\oxb+0*\dx,\by+1*\dy) {buffer};

    \node[focus1] (0bb) at  (\ox+0*\dx,2.5*\dy) {};
    \node[word]  (0b3) at (\ox+0*\dx,\by-0.5*\dy) {};
    \node[word]  (0b2) at (\ox+0*\dx,\by+1.5*\dy) {down};
    \node[word]  (0b1) at (\ox+0*\dx,\by+2.5*\dy) {sat};
    \node[bbox] (0bb) at  (\ox+0*\dx,\by+1.0*\dy) {};
    
    \node[label]  (sl) at (\ox-0.35*\oxb+0*\dx,\sy+0.5*\dy) {stack};
    
    \node[focus2] (0sb) at  (\ox+0*\dx,\sy-0.5*\dy) {};
    \node[word]  (0s1) at (\ox+0*\dx,\sy-1*\dy) {cat};
    \node[word]  (0s2) at (\ox+0*\dx,\sy+0*\dy) {the};
    \node[word]  (0s3) at (\ox+0*\dx,\sy+1*\dy) {};
    \node[sbox] (0sb) at  (\ox+0*\dx,\sy+0.5*\dy) {};
    
    \node[comp] (0c) at  (\ox+0.5*\oxb,\sy-1.5*\dy) {composition};
    
    \node[tracker] (0t) at  (\ox+0*\dx,5*\dy) {tracking};
    \node[softmax] (0sm) at  (\ox+3.25*\dx,6.25*\dy) {transition};
    \node[result] (0so) at  (\ox+3.25*\dx,8.5*\dy) {\reduce};
    
    \def\ox{1*\oxb}

    \node[focus1] (1bb) at  (\ox+0*\dx,2.5*\dy) {};
    \node[word]  (1b3) at (\ox+0*\dx,\by-0.5*\dy) {};
    \node[word]  (1b2) at (\ox+0*\dx,\by+1.5*\dy) {down};
    \node[word]  (1b1) at (\ox+0*\dx,\by+2.5*\dy) {sat};
    \node[bbox] (1bb) at  (\ox+0*\dx,\by+1*\dy) {};
    
    \node[focus2] (1sb) at  (\ox+0*\dx,\sy-0.5*\dy) {};
    \node[word]  (1s1) at (\ox+0*\dx,\sy-1*\dy) {the cat};
    \node[word]  (1s2) at (\ox+0*\dx,\sy+0*\dy) {};
    \node[word]  (1s3) at (\ox+0*\dx,\sy+1*\dy) {};
    \node[sbox] (1sb) at  (\ox+0*\dx,\sy+0.5*\dy) {};
    
    \node[compoff] (1c) at  (\ox+0.5*\oxb,\sy-1.5*\dy) {composition};
    
    \node[tracker] (1t) at  (\ox+0*\dx,5*\dy) {tracking};
    \node[softmax] (1sm) at  (\ox+3.25*\dx,6.25*\dy) {transition};
    \node[result] (1so) at  (\ox+3.25*\dx,8.5*\dy) {\shift};
     
    \def\ox{2*\oxb}

    \node[focus1] (2bb) at  (\ox+0*\dx,2.5*\dy) {};
    \node[word]  (2b3) at (\ox+0*\dx,\by-0.5*\dy) {};
    \node[word]  (2b2) at (\ox+0*\dx,\by+1.5*\dy) {};
    \node[word]  (2b1) at (\ox+0*\dx,\by+2.5*\dy) {down};
    \node[bbox] (2bb) at  (\ox+0*\dx,\by+1*\dy) {};
    
    \node[focus2] (2sb) at  (\ox+0*\dx,\sy-0.5*\dy) {};
    \node[word]  (2s1) at (\ox+0*\dx,\sy-1*\dy) {sat};
    \node[word]  (2s2) at (\ox+0*\dx,\sy+0*\dy) {the cat};
    \node[word]  (2s3) at (\ox+0*\dx,\sy+1*\dy) {};
    \node[sbox] (2sb) at  (\ox+0*\dx,\sy+0.5*\dy) {};
   
    \node[tracker] (2t) at  (\ox+0*\dx,5*\dy) {tracking};

    
    \pgfsetarrowsend{latex}
    \tikzstyle{fwd} = [draw=black, line width=1pt]
    \tikzstyle{gated} = [draw=black!50, line width=0.8pt]

    \draw [fwd] (0sb) -- (0t);
    \draw [fwd] (0bb) -- (0t);
    \draw [fwd] (0t) -- (0sm);
    \draw [fwd] (0sm) -- (0so);
    \draw [fwd] (0sb) -- (0c);
    
    \draw [fwd] (0t) -- (1t);
    \draw [fwd] (0t) to[out=50,in=-175] (0c);
    \draw [fwd] (0sb) -- (1sb);
    \draw [fwd] (0bb) -- (1bb);
    \draw [fwd] (0so) to[out=5,in=-140] (1sb);
    \draw [fwd] (0so) to[out=-5,in=170] (1bb);
    \draw [gated] (0bb) to[out=15,in=-125] (1sb);
    \draw [fwd] (0c) -- (1sb);

    \draw [fwd] (1sb) -- (1t);
    \draw [fwd] (1bb) -- (1t);
    \draw [fwd] (1t) -- (1sm);
    \draw [fwd] (1sm) -- (1so);
    \draw [fwd] (1sb) -- (1c);
    
    \draw [fwd] (1t) -- (2t);
    \draw [fwd] (1t) to[out=50,in=-175] (1c);
    \draw [fwd] (1sb) -- (2sb);
    \draw [fwd] (1bb) -- (2bb);
    \draw [fwd] (1so) to[out=5,in=-140] (2sb);
    \draw [fwd] (1so) to[out=-5,in=170] (2bb);
    \draw [fwd] (1bb) to[out=15,in=-125] (2sb);
    \draw [gated] (1c) -- (2sb);

    \draw [fwd] (2sb) -- (2t);
    \draw [fwd] (2bb) -- (2t);


  \end{tikzpicture}}
  
 \caption{The SPINN model unrolled for two transitions during the processing of the sentence \word{the cat sat down}. `Tracking', `transition', and `composition' are neural network layers. Gray arrows indicate connections which are blocked by a gating function.}\label{fig:model:1d}
  
\end{subfigure}\\\\\\
\begin{subfigure}[t]{\textwidth}
\centering
\scalebox{0.5}{%
\begin{tikzpicture}
    \def\dx{20pt}
    \def\dy{11pt}
    \def\sy{7*\dy}
    \def\oxb{5.5*\dx}
{'fraction_non_alphanumeric': 0.2702373135728356, 'fraction_numerical': 0.06894287590282337, 'mean_word_length': 2.8560578661844485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2722572', 'n_tokens_mistral': 5258, 'n_tokens_neox': 5106, 'n_words': 828}