<commit_before>% This is the plain TeX format that's described in The TeXbook.
% N.B.: A version number is defined at the very end of this file;
%       please change that number whenever the file is modified!
% And don't modify the file unless you change its name:
%       Everybody's "plain.tex" file should be the same, worldwide.

% Unlimited copying and redistribution of this file are permitted as long
% as this file is not modified. Modifications are permitted, but only if
% the resulting file is not named plain.tex.

\catcode`\{=1 % left brace is begin-group character
\catcode`\}=2 % right brace is end-group character
\catcode`\$=3 % dollar sign is math shift
\catcode`\&=4 % ampersand is alignment tab
\catcode`\#=6 % hash mark is macro parameter character
\catcode`\^=7 \catcode`\^^K=7 % circumflex and uparrow are for superscripts
\catcode`\_=8 \catcode`\^^A=8 % underline and downarrow are for subscripts
\catcode`\^^I=10 % ascii tab is a blank space
\chardef\active=13 \catcode`\~=\active % tilde is active
\catcode`\^^L=\active \outer\def^^L{\par}% ascii form-feed is "\outer\par"<commit_msg>Update p.tex to where we can now parse to<commit_after>% This is the plain TeX format that's described in The TeXbook.
% N.B.: A version number is defined at the very end of this file;
%       please change that number whenever the file is modified!
% And don't modify the file unless you change its name:
%       Everybody's "plain.tex" file should be the same, worldwide.

% Unlimited copying and redistribution of this file are permitted as long
% as this file is not modified. Modifications are permitted, but only if
% the resulting file is not named plain.tex.

\catcode`\{=1 % left brace is begin-group character
\catcode`\}=2 % right brace is end-group character
\catcode`\$=3 % dollar sign is math shift
\catcode`\&=4 % ampersand is alignment tab
\catcode`\#=6 % hash mark is macro parameter character
\catcode`\^=7 \catcode`\^^K=7 % circumflex and uparrow are for superscripts
\catcode`\_=8 \catcode`\^^A=8 % underline and downarrow are for subscripts
\catcode`\^^I=10 % ascii tab is a blank space
\chardef\active=13 \catcode`\~=\active % tilde is active
\catcode`\^^L=\active \outer\def^^L{\par} % ascii form-feed is "\outer\par"

\message{Preloading the plain format: codes,}

% We had to define the \catcodes right away, before the message line,
% since \message uses the { and } characters.
% When INITEX (the TeX initializer) starts up,
% it has defined the following \catcode values:
% \catcode`\^^@=9 % ascii null is ignored
% \catcode`\^^M=5 % ascii return is end-line
% \catcode`\\=0 % backslash is TeX escape character
% \catcode`\%=14 % percent sign is comment character
% \catcode`\ =10 % ascii space is blank space
% \catcode`\^^?=15 % ascii delete is invalid
% \catcode`\A=11 ... \catcode`\Z=11 % uppercase letters
% \catcode`\a=11 ... \catcode`\z=11 % lowercase letters
% all others are type 12 (other)

% Here is a list of the characters that have been specially catcoded:
{'fraction_non_alphanumeric': 0.11432369557992689, 'fraction_numerical': 0.014955134596211365, 'mean_word_length': 3.950657894736842, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2022708', 'n_tokens_mistral': 967, 'n_tokens_neox': 924, 'n_words': 396}