<jupyter_start><jupyter_text>Dataset  Importing it<jupyter_code>data = pd.read_csv("G:\OneDrive - University of Edinburgh\Poem Generation\WebScrapping-PoetryFoundation\PoetryFoundationData.csv")
data = data.drop(columns=["Unnamed: 0"])
data.head(5)<jupyter_output><empty_output><jupyter_text>Cleaning the dataset<jupyter_code>def clean_text(string):
    string  = str(string).upper()
    string  = string.replace("-", " ")
    string  = string.replace("—", " ")
    string  = string.replace(".", " . ")
    return str(string).replace("\r\r\n"," NEWLINE\n ")

data = data.applymap(clean_text)

poems = data.Poem.map(lambda x: nltk.tokenize.word_tokenize(x,preserve_line =True))
#For Faster Testing
poem_sample = poems[:2000]<jupyter_output><empty_output><jupyter_text>Adding Syllables Count Importing the Syllables dictionary<jupyter_code>syllables = pd.read_csv("Syllable dict.txt","  ", engine="python")
syllables = syllables.rename(columns = {"A":"words","AH0":"syllable_count"})
syllables.head()<jupyter_output><empty_output><jupyter_text>Building the Syllables dictionary<jupyter_code>def number_of_syllables(string):
    count = 0
    for ch in string:
        if(ch in "012"):
            count=count+1
    return(count)

syllables.syllable_count = (syllables.syllable_count.map(number_of_syllables))
syllable_dict = dict([(word, syllable)for word,syllable in zip(syllables.words,syllables.syllable_count)])
#Some other basic additions to it manually
syllable_dict["A"] = 1
syllable_dict["NEWLINE"] = "NEWLINE"
syllable_dict[","] = "punc"
syllable_dict["."] = "punc"<jupyter_output><empty_output><jupyter_text>Tagging the poems with the syllables dictionary Tags the words present in the dictionary and returns the words not present in it<jupyter_code>def tag_syllables(poem, untagged):
    tagged_poem = []
    for token in poem:
        if(token in syllable_dict):
            tag = syllable_dict[token]
            tagged_poem = tagged_poem + [(token,tag)] 
        else:
            untagged += [token]
    return(tagged_poem, untagged)<jupyter_output><empty_output><jupyter_text>The missing words problem<jupyter_code>b = []
for i in poems:
    a,b = tag_syllables(i,b)
#set(b)
len(set(b))
list(set(untagged))<jupyter_output><empty_output><jupyter_text>Using CMU Building to build more dictionaries<jupyter_code>File_object = open("missing_words.txt","w")
c = []
for word in (list(set(untagged))):
    try:
        #print(word)
        File_object.writelines(word+"\n")
    except Exception as e:
        c = c+[word]
File_object.close()<jupyter_output><empty_output><jupyter_text>Using the dictionaries to find the counts<jupyter_code>dict1 = pd.read_csv("dict1.txt","\t", engine="python")
dict1 = dict1.rename(columns={"SCYTHING":"words","S IH DH AH NG":"syllable_count"})
dict2 = pd.read_csv("dict2.txt","\t", engine="python")
dict2 = dict2.rename(columns={"TREATERS/":"words","T R IY T ER Z":"syllable_count"})
dict3 = pd.read_csv("dict3.txt","\t", engine="python")
dict3 = dict3.rename(columns={"NOMMO":"words","N AA M OW":"syllable_count"})
dict4 = pd.read_csv("dict4.txt","\t", engine="python")
dict4 = dict4.rename(columns={"UNINFORMD":"words","Y UW N AH N F AO R M D":"syllable_count"})

dictionary = ((dict1.append(dict2)).append(dict3)).append(dict4)
dictionary.head()<jupyter_output><empty_output><jupyter_text>Importing the phones to identify syllables<jupyter_code>phones = pd.read_csv("phones.txt","\t", engine="python")
phones = phones.rename(columns={"AA":"symbol","vowel":"meaning"})
phones = phones[phones["meaning"]=="vowel"]
vowels = list(phones.symbol)
vowels = vowels + ["AA"]
print(vowels)
def syllables_count(string):
    l = str(string).split(" ")
    count = 0
    for a in l:
        if(a in vowels):
            count+=1
    return count
dictionary.syllable_count = dictionary.syllable_count.map(syllables_count)
dict_database = syllables.append(dictionary)
syllable_dict = dict([(word, syllable)for word,syllable in zip(dict_database.words,dict_database.syllable_count)])
#Some other basic additions to it manually
syllable_dict["A"] = 1
syllable_dict["NEWLINE"] = "NEWLINE"
syllable_dict[","] = "punc"
syllable_dict["."] = "punc"<jupyter_output><empty_output><jupyter_text>Tagging poems with the updated dictionary!<jupyter_code>def syllables_poem(poem, untagged):
    syllable_poem = []
    for token in poem:
        if(token in syllable_dict):
            tag = syllable_dict[token]
            syllable_poem = syllable_poem + [tag] 
        else:
            untagged += [token]
    return(syllable_poem, untagged)

untagged = []
syllable_poems = []
for i in poems:
    temp,untagged = syllables_poem(i,untagged)
    syllable_poems = syllable_poems+[temp]
#set(b)
len(set(untagged))
set(untagged)
from Models import nGrams as n
import Models
a = Models.nGrams()
dicTri = {}
for i in syllable_poems:
    dicTri = n.TriGram(dicTri, i)

dic4 = {}
for i in syllable_poems:
    dic4 = a.NGram(dic4, i, 4)
prev_word1 = 2
prev_word2 = 2
prev_word3 = 2
prev_word4 = 2
prev_word5 = 3
prev_word6 = 2
prev_word7 = 3

generate_str = str(prev_word1)+" "+str(prev_word2) + " " +str(prev_word3) + " "+ str(prev_word4) +" "+str(prev_word5) + " " +str(prev_word6) + " " +str(prev_word7)

for i in range(50):
    next_word  = n.next_word([prev_word1, prev_word2,prev_word3,prev_word4, prev_word5,prev_word6,prev_word7], dic8, 8)
    prev_word1 = prev_word2
    prev_word2 = prev_word3
    prev_word3 = prev_word4
    prev_word4 = prev_word5
    prev_word5 = prev_word6
    prev_word6 = prev_word7
    prev_word7 = next_word
    generate_str = generate_str+" "+ str(next_word)

print(generate_str)
dic8 = {}
for i in syllable_poems:
    dic8 = a.NGram(dic8, i, 8)
next_word([11], dicBi, 2)
def next_word_prob(words, dic, nGram):
    Max = -inf
    next_word = ""
    dictionary = dic
    for word in words:
        dictionary = dictionary[word]
    
    if(not(list(dictionary.values())[0]<1)):
        total_counts = 0
        for i,j in list(dictionary.items()):
            total_counts = total_counts+j
        print(total_counts)
        items = list(dictionary.items())
        alpha = 0.1*total_counts
        for i,j in items:
            dictionary[i] = ((j+alpha)/(total_counts + alpha*len(items)))

    for i,j in list(dictionary.items()):
        if(j>Max):
            Max = j
            next_word = i
    return(next_word, Max)
dicBi = {}
for i in syllable_poems:
    dicBi = n.BiGram(dicBi, i)
for j in range(13):
    print(j, next_word([j], dicBi, 2))
dicBi
#dicTri = {}
#for i in syllable_poems:
#    dicTri = n.TriGram(dicTri, i)
    
next_word(["NEWLINE",4], dicTri,3)
dicTri
from math import log
dict_database
dict_database[dict_database.syllable_count ==13]
dictSyllable = {}
for i in dict_database.syllable_count.unique():
    dictSyllable[i] = list((dict_database[dict_database.syllable_count ==i]).words)
from math import inf
import Data
Data.Poems
help(Data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13989338712001154, 'fraction_numerical': 0.014983431782163954, 'mean_word_length': 4.66231647634584, 'pattern_counts': {'":': 12, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '619717', 'license': 'MIT', 'n_tokens_mistral': 2669, 'n_tokens_neox': 2573, 'n_words': 496, 'path': '.ipynb_checkpoints/Syllable-checkpoint.ipynb', 'repo_name': 'TGDivy/Poem-Generation-Basic', 'size': '6941'}