<commit_before>\documentclass[conference]{IEEEtran}

\usepackage{verbatim}
\usepackage{framed}
\usepackage{url}
\usepackage{flushend}

\begin{document}

\title{CSE6339 -- The Big Assignment\vspace{-14pt}}
\author{<NAME> -- <EMAIL>, <EMAIL>\vspace{4pt} \\ March, 2014}

\maketitle


\begin{abstract}
Solutions to the second assignment of the course CSE6339 3.0 Introduction to 
Computational Linguistics. Instructor: Prof. <NAME>, Department of 
Electrical Engineering \& Computer Science, York University, Canada.
\end{abstract}

\section{Introduction}


\section{Program Documentation}


\section{Solutions}

\subsection{Problem 1a}

\begin{quote}
``Simulate the straightforward monkey problem. Let the program run long enough to 
give a meaningful estimate of the yield of words. The result will provide a 
useful comparison with later forms of the problem.''
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1a.py}
\end{framed}


\subsection{Problem 1b}

\begin{quote}
``Use the data in Table~\ref{tab:hamlet} to simulate the first-order monkey problem. 
Again let the program run long enough to give a meaningful estimate of the yield 
of words to permit comparison with other results on relative word yield. Try running
this simulation program against other corpora.''
\end{quote}

\begin{table}
\caption{\hspace{2em}Character Distribution from Act III of Hamlet. \newline
Note: 35,224 characters, a small corpus.\label{tab:hamlet}}
\vspace{-10pt}
\begin{center}
\begin{tabular}{crcrcr}
\hline
Char. & Freq. & Char. & Freq. & Char. & Freq. \\
\hline
space & 6,934  & r     & 1,593  & p     & 433   \\
e     & 3,277  & l     & 1,238  & b     & 410   \\
o     & 2,578  & d     & 1,099  & v     & 309   \\
t     & 2,557  & u     & 1,014  & k     & 255   \\
a     & 2,043  & m     & 889   & '     & 203   \\
s     & 1,856  & y     & 783   & j     & 34    \\
h     & 1,773  & f     & 629   & q     & 27    \\
n     & 1,741  & c     & 584   & x     & 21    \\
i     & 1,736  & g     & 478   & z     & 14    \\
\hline
\end{tabular}
\end{center}
\end{table}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1b.py}
\end{framed}


\subsection{Problem 1c}

\begin{quote}
``Use the data supplied, data listed in Table~\ref{tab:books}, to simulate the 
second-order and third order Bronte monkey problem. Again let the program run 
long enough to give a meaningful estimate of the yield of words to permit 
comparison with other results on relative word yield. Try running this 
simulation program against other authors listed.''
\end{quote}

\begin{table}
\caption{Data made available for this assignment.\label{tab:books}}
\vspace{-10pt}
\begin{center}
\begin{tabular}{llr}
\hline
Author & Title & Chars. \\
\hline
<NAME> & A Christmas Carol & 168,920 \\
<NAME> & A Tale of Two Cities & 773,928 \\
<NAME> & Wuthering Heights & 662,869 \\
<NAME> & <NAME> & 384,300 \\
<NAME> & <NAME> & 1,051,336 \\
<NAME> & Tarzan of the Apes & 492,783 \\
<NAME> & Warlord of Mars & 319,854 \\
<NAME> & The People that Time Forgot & 212,736 \\
<NAME> & The Land that Time Forgot & 205,762 \\
<NAME> & King Solomon's Mines & 454,437 \\
<NAME> & <NAME> & 474,725 \\
<NAME> & Alice's Adventures in Wonderland & 148,580 \\
<NAME> & Through the Looking Glass & 167,015 \\
<NAME> & Legend of Sleepy Hollow & 69,164 \\
Sir <NAME> & The Adventures of Sherlock Holmes & 575,559 \\
Sir <NAME> & The Lost World & 432,766 \\
Sir <NAME> & The Hound of the Baskervilles & 327,348 \\
Sir <NAME> & Tales of Terror and Mystery & 421,264 \\
<NAME> & Adventures of Huckleberry Finn & 578,345 \\
<NAME> & The Adventures of Tom Sawyer & 397,076 \\
<NAME>ain & A Connecticut Yankee in & 657,430 \\
           & King Arthur's Court & \\
<NAME> & The Prince & 286,854 \\
<NAME> & War of the Worlds & 346,458 \\
<NAME> & The Time Machine & 182,881 \\
F. Kafka & Metamorphosis & 122,139 \\
F. Kafka & The Trial & 462,372 \\
R. Kipling & The Jungle Book & 279,771 \\
\hline
\end{tabular}
\end{center}
\end{table}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1c.py}
\end{framed}

\subsection{Problem 1d}

\begin{quote}
``Investigate the effects of resolution on monkey literacy in the simulation. 
For example round off the matrix elements to the smallest number of places --
or use an equivalent means to reduce the number of keys on the typewriters.''
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1d.py}
\end{framed}

\subsection{Problem 1e}

\begin{quote}
``Write a routine to compute correlation matrices of the type shown in the 
handout~\cite{Bennett1976} from data supplied (the books shown in Table~\ref{tab:books}).''
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1e.py}
\end{framed}

\subsection{Problem 1f}

\begin{quote}
''Using the algorithm in the handout~\cite{Bennett1976} with the pair-correlation matrix 
generated from Irving (book shown in Table~\ref{tab:books}), compute the most 
probable digraph path, which starts with the letter 't'. Compare the result with 
that given in the handout for Poe's `The Gold Bug'''.
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1f.py}
\end{framed}

\subsection{Problem 1g}

\begin{quote}
``Design and implement an experiment using data from the books shown in Table~\ref{tab:books}
that might be used to perform author attribution. Discuss your solution and provide reasons 
why it is likely or not likely to solve the problem definitively.''
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont<commit_msg>Set a number to each book the report.<commit_after>\documentclass[conference]{IEEEtran}

\usepackage{verbatim}
\usepackage{framed}
\usepackage{url}
\usepackage{flushend}

\begin{document}

\title{CSE6339 -- The Big Assignment\vspace{-14pt}}
\author{<NAME> -- <EMAIL>, <EMAIL>\vspace{4pt} \\ March, 2014}

\maketitle


\begin{abstract}
Solutions to the second assignment of the course CSE6339 3.0 Introduction to 
Computational Linguistics. Instructor: Prof. <NAME>, Department of 
Electrical Engineering \& Computer Science, York University, Canada.
\end{abstract}

\section{Introduction}


\section{Program Documentation}


\section{Solutions}

\subsection{Problem 1a}

\begin{quote}
``Simulate the straightforward monkey problem. Let the program run long enough to 
give a meaningful estimate of the yield of words. The result will provide a 
useful comparison with later forms of the problem.''
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1a.py}
\end{framed}


\subsection{Problem 1b}

\begin{quote}
``Use the data in Table~\ref{tab:hamlet} to simulate the first-order monkey problem. 
Again let the program run long enough to give a meaningful estimate of the yield 
of words to permit comparison with other results on relative word yield. Try running
this simulation program against other corpora.''
\end{quote}

\begin{table}
\caption{\hspace{2em}Character Distribution from Act III of Hamlet. \newline
Note: 35,224 characters, a small corpus.\label{tab:hamlet}}
\vspace{-10pt}
\begin{center}
\begin{tabular}{crcrcr}
\hline
Char. & Freq. & Char. & Freq. & Char. & Freq. \\
\hline
space & 6,934  & r     & 1,593  & p     & 433   \\
e     & 3,277  & l     & 1,238  & b     & 410   \\
o     & 2,578  & d     & 1,099  & v     & 309   \\
t     & 2,557  & u     & 1,014  & k     & 255   \\
a     & 2,043  & m     & 889   & '     & 203   \\
s     & 1,856  & y     & 783   & j     & 34    \\
h     & 1,773  & f     & 629   & q     & 27    \\
n     & 1,741  & c     & 584   & x     & 21    \\
i     & 1,736  & g     & 478   & z     & 14    \\
\hline
\end{tabular}
\end{center}
\end{table}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1b.py}
\end{framed}


\subsection{Problem 1c}

\begin{quote}
``Use the data supplied, data listed in Table~\ref{tab:books}, to simulate the 
second-order and third order Bronte monkey problem. Again let the program run 
long enough to give a meaningful estimate of the yield of words to permit 
comparison with other results on relative word yield. Try running this 
simulation program against other authors listed.''
\end{quote}

\begin{table}
\caption{Data made available for this assignment.\label{tab:books}}
\vspace{-10pt}
\begin{center}
\begin{tabular}{r@{\hspace{1.1em}}l@{\hspace{1.1em}}l@{\hspace{0.75em}}r}
\hline
No. & Author & Title & Chars. \\
\hline
1  & <NAME> & A Christmas Carol & 168,925 \\
2  & <NAME> & A Tale of Two Cities & 773,928 \\
3  & <NAME> & Wuthering Heights & 662,869 \\
4  & <NAME> & Agnes Grey & 384,300 \\
5  & <NAME> & <NAME> & 1,051,336 \\
6  & <NAME> & Tarzan of the Apes & 492,783 \\
7  & <NAME> & Warlord of Mars & 319,854 \\
8  & <NAME> & The People that Time Forgot & 212,736 \\
9  & <NAME> & The Land that Time Forgot & 205,762 \\
10 & <NAME> & King Solomon's Mines & 454,437 \\
11 & <NAME> & <NAME> & 474,725 \\
12 & <NAME> & Alice's Adventures in Wonderland & 148,580 \\
13 & <NAME> & Through the Looking Glass & 167,015 \\
14 & <NAME> & Legend of Sleepy Hollow & 69,164 \\
15 & Sir <NAME> & The Adventures of Sherlock Holmes & 575,574 \\
16 & Sir <NAME> & The Lost World & 432,766 \\
17 & Sir <NAME> & The Hound of the Baskervilles & 327,348 \\
18 & Sir <NAME> & Tales of Terror and Mystery & 421,264 \\
19 & <NAME> & Adventures of Huckleberry Finn & 578,345 \\
20 & <NAME> & The Adventures of Tom Sawyer & 397,076 \\
21 & <NAME> & A Connecticut Yankee in & 657,430 \\
   &          & King Arthur's Court & \\
22 & <NAME> & The Prince & 286,854 \\
23 & <NAME> & War of the Worlds & 346,458 \\
24 & <NAME> & The Time Machine & 182,881 \\
25 & <NAME> & Metamorphosis & 122,139 \\
26 & <NAME> & The Trial & 462,372 \\
27 & <NAME> & The Jungle Book & 279,771 \\
\hline
\end{tabular}
\end{center}
\end{table}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1c.py}
\end{framed}

\subsection{Problem 1d}

\begin{quote}
``Investigate the effects of resolution on monkey literacy in the simulation. 
For example round off the matrix elements to the smallest number of places --
or use an equivalent means to reduce the number of keys on the typewriters.''
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1d.py}
\end{framed}

\subsection{Problem 1e}

\begin{quote}
``Write a routine to compute correlation matrices of the type shown in the 
handout~\cite{Bennett1976} from data supplied (the books shown in Table~\ref{tab:books}).''
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1e.py}
\end{framed}

\subsection{Problem 1f}

\begin{quote}
''Using the algorithm in the handout~\cite{Bennett1976} with the pair-correlation matrix 
generated from Irving (book shown in Table~\ref{tab:books}), compute the most 
probable digraph path, which starts with the letter `t'. Compare the result with 
that given in the handout for Poe's `The Gold Bug'''.
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
\verbatiminput{../code/problem1f.py}
\end{framed}

\subsection{Problem 1g}

\begin{quote}
``Design and implement an experiment using data from the books shown in Table~\ref{tab:books}
that might be used to perform author attribution. Discuss your solution and provide reasons 
why it is likely or not likely to solve the problem definitively.''
\end{quote}

\begin{framed}
\fontsize{5.65}{6.78}\selectfont
{'fraction_non_alphanumeric': 0.13494539781591264, 'fraction_numerical': 0.06378921823539609, 'mean_word_length': 3.400839054157132, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4073168', 'n_tokens_mistral': 4514, 'n_tokens_neox': 3902, 'n_words': 1302}