<commit_before><commit_msg>Break the master file into its components<commit_after>#/bin/bash

# Determine the length of the file, since we'll need this later. It's so that we can search
# the first half of the file for Limited Partnerships and the second half for Limited Liability
# Companies.
#
# TO DO: We're faking this right now, manually setting the length of one half of the file. That's
# because rounding numbers in Bash turns out to be surprisingly difficult.
FILE_LENGTH=`wc -l cisbemon.txt |cut -d " " -f 2`
HALF_LENGTH=891210

# File 1, in the order of the master file
echo Table File
awk 'length() == 65' cisbemon.txt > tables.txt;

# File 2, in the order of the master file
echo Corporate File
awk 'length() == 674' cisbemon.txt > corporate.txt;

# File 3, in the order of the master file
echo Limited Partnership File
head -$HALF_LENGTH cisbemon.txt | awk 'length() == 508' > lp.txt;

# File 4, in the order of the master file
echo Corporate/Limited Partnership/Limited Liability Company File
awk 'length() == 182' cisbemon.txt > amendments.txt;

# File 5, in the order of the master file
echo Corporate Officer File
awk 'length() == 95' cisbemon.txt > officers.txt;

# File 6, in the order of the master file
echo Corporate/Limited Partnership/Limited Liability Company Name File
awk 'length() == 120' cisbemon.txt > name.txt;

# File 7, in the order of the master file
echo Merger File
awk 'length() == 126' cisbemon.txt > merger.txt;

# File 8, in the order of the master file
echo Corporate/Limited Partnership/Limited Liability Company Reserved/Registered Name File
awk 'length() == 335' cisbemon.txt > registered_names.txt;

# File 9, in the order of the master file
#
# The line length is exactly the same as the LP file, so we can't simply create the file based
# on line length. Instead, we count the number of lines in files 1, 2, and 3, and only look at
# the files in the file subsequent to that.
echo Limited Liability Company File
tail -$HALF_LENGTH cisbemon.txt |awk 'length() == 508' > llc.txt;

{'fraction_non_alphanumeric': 0.08386778490379872, 'fraction_numerical': 0.021706956092747903, 'mean_word_length': 4.22680412371134, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '608973', 'n_tokens_mistral': 663, 'n_tokens_neox': 589, 'n_words': 298}