<jupyter_start><jupyter_text>Code for finding the mismatched titles<jupyter_code>search_title = lambda df, title, regex=False: df[df["Title"].str.contains(title, regex=regex, case=False, flags=re.IGNORECASE)]
#Not surprisingly, the statistical reports are not included
search_title(summary, "Statistical Report")
search_title(summary, "the Choice is Yours")
search_title(summary, "The Great and Wonderful Love")
search_title(summary, "Be Ye Strong from Henceforth")
search_title(summary, "On Zion")
search_title(summary, "Have Ye Done This?")
search_title(summary, "Pioneers of the Future:")["Title"].item()
search_title(summary, "Thy Church, O Lord")
search_title(summary, "Come unto Christ, and be perfected in Him")
search_title(summary, "The Refiner")
search_title(summary, "If Christ Had My Opportunities")
search_title(summary, "Were Our Examples")
search_title(summary, "She Is Not Afraid of the Snow for Her Household")
search_title(summary, "Divine Design")
search_title(summary, "Do Unto Others")
search_title(summary, "Counsellors There Is Safety")
search_title(summary, "Can Be Destroyed through Spirituality")
search_title(summary, "But If Not")
search_title(summary, "Behold, the Enemy Is Combined")
search_title(summary, "But Be Ye Doers of the Word")
search_title(summary, "There Is the Light")
search_title(summary, "God Moves in a Mysterious Way His Wonders to Perform")["Title"].item()
search_title(summary, "Salvation for the Dead")
search_title(summary, "To Be Learned Is Good If")
search_title(summary, "Judge Not, That Ye Be Not Judged")
search_title(summary, "The Standard of Truth Has Been Erected")
search_title(summary, "Ponder the Path of Thy Feet")
search_title(summary, "Draw Near unto Me")
search_title(summary, "Pour You Out a Blessing")
search_title(summary, "Mother Heart")
search_title(summary, "Church Welfare Services Basic Principles")
search_title(summary, "Relief Societys Role in Welfare Services")
search_title(summary, "A Proclamation to the World")
search_title(summary, "Be Ye Clean that Bear the Vessels of the Lord")
search_title(summary, "Thou Mayest Choose for Thyself")
search_title(summary, "Give Heed unto the Prophet")
search_title(summary, "My Words")
search_title(summary, "Seeing the Five A")
search_title(summary, "By Way of Invitation")
search_title(summary, "Then Teach")
search_title(summary, "Did I tell You")
search_title(summary, "That I May Heal You")
search_title(summary, "Solutions from the Scriptures")
search_title(summary, "Repent of")
search_title(summary, "I Will Go and Do…")
search_title(summary, "Even As I Also Overcame")
search_title(summary, "What Went Ye Out… to See?")
search_title(summary, "Except the Lord Build the House…")
search_title(summary, "Report")
summary.loc[2838, "Title"]
search_title(summary, "The Reconstitution")
search_title(summary, "for Boys")
search_title(summary, "Anxiously Engaged")
search_title(summary, "Her Children Arise Up")
#%run preprocess.py
#create_train_test_split()
testdf = pd.DataFrame.from_dict({"a":["hello", "world"], "b":["engaged", "redeemer"]})
testdf
testdf.loc[testdf["a"] == "hello", "a"] = "hi"
testdf
(testdf["a"]  == "world") & (testdf["b"]  == "redeemer")
testdf.loc[(testdf["a"]  == "world") & (testdf["b"]  == "redeemer")]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12941534713763703, 'fraction_numerical': 0.001218026796589525, 'mean_word_length': 8.074585635359115, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '385044', 'license': 'MIT', 'n_tokens_mistral': 1076, 'n_tokens_neox': 1032, 'n_words': 337, 'path': 'data_processing/title_alignment.ipynb', 'repo_name': 'lodeous/gencon-nlp', 'size': '3284'}