<jupyter_start><jupyter_text>**Word embedding**<jupyter_code>import pandas as pd
import numpy as np
import json
import glob

#Gensim
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel

#spacy
import spacy
from nltk.corpus import stopwords

#vis
#import pyLDAvis
#import pyLDAvis.gensim
import pyLDAvis.gensim_models as gensimvis
#pyLDAvis.enable_notebook()

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)<jupyter_output><empty_output><jupyter_text>**Converting Chapter Wise_Master design in Supervised-ML to Java Script Object Notation**<jupyter_code>df = pd.read_csv("Supervised_ML_Master/Para_Wise_M_L-Model_OF_NOBEL_QURAN.csv", header = None, encoding = "ISO-8859-1")
df.drop([0],axis=0,inplace=True)
df = pd.DataFrame(df).rename(columns = {0:"Surah",1:'Ayat',2:'Para',3:'Data'})
#Data Snapshot
df.head()
#Reinforcement_ML_by_Numerical_Analysis
df.to_json("Reinforcement_ML_by_Numerical_Analysis/Numerical_Analysis_On_NOBEL_QURAN.json")
df1=pd.read_json("Reinforcement_ML_by_Numerical_Analysis/Numerical_Analysis_On_NOBEL_QURAN.json")
df1.head()<jupyter_output><empty_output><jupyter_text>**Handy function to optimize json**<jupyter_code>def load_data(file):
    with open (file, "r", encoding="utf-8") as f:
        data = json.load(f) 
    return (data)

def write_data(file, data):
    with open (file, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=4)
stopwords= stopwords.words("english")<jupyter_output><empty_output><jupyter_text>**preview of jason file create above**<jupyter_code>data = load_data("Reinforcement_ML_by_Numerical_Analysis/Numerical_Analysis_On_NOBEL_QURAN.json")["Data"]
print (data)<jupyter_output>{'1': 'In the name of Allah Most Gracious Most Merciful.', '2': 'Praise be to Allah the Cherisher and Sustainer of the worlds;', '3': 'Most Gracious Most Merciful;', '4': 'Master of the Day of Judgment.', '5': 'Thee do we worship and Thine aid we seek.', '6': 'Show us the straight way', '7': 'The way of those on whom Thou hast bestowed Thy Grace those whose (portion) is not wrath and who go not astray.', '8': 'A.L.M.', '9': 'This is the Book; in it is guidance sure without doubt to those who fear Allah;', '10': 'Who believe in the Unseen are steadfast in prayer and spend out of what We have provided for them;', '11': 'And who believe in the Revelation sent to thee and sent before thy time and (in their hearts) have the assurance of the Hereafter.', '12': 'They are on (true) guidance from their Lord and it is these who will prosper.', '13': 'As to those who reject Faith it is the same to them whether thou warn them or do not warn them; they will not believe.', '14': 'Allah hath set a se[...]<jupyter_text>**doc container missing text when passing through nlp.****Insha'ALLAH soon I get solution of it.This code is only for reference to make clusters Of NOBEL_QURAN**<jupyter_code>def lemmatization(data):#,allowed_postags=["NOUN", "ADJ", "VERB", "ADV"]):
    nlp = spacy.load("en_core_web_sm", disable=["parser", "ner"])
    texts_out = []
    for texts in data:
        doc = nlp(texts)
        new_text = []
        for token in doc:
            if token.pos_ in data:#allowed_postags:
                new_text.append(token.lemma_)
        final = " ".join(new_text)
        texts_out.append(final)
    return (texts_out)


lemmatized_texts = lemmatization(texts)
print (lemmatized_texts)<jupyter_output><empty_output><jupyter_text>**preprocessing by gensim**<jupyter_code>def gen_words(data):
    final = []
    for text in Data:
        new = gensim.utils.simple_preprocess(text, deacc=True)
        final.append(new)
    return (final)

data_words = gen_words(data)

print (data_words)<jupyter_output><empty_output><jupyter_text>**creating word vectors**<jupyter_code>id2word = corpora.Dictionary(data_words)

corpus = []
for text in data_words:
    new = id2word.doc2bow(text)
    corpus.append(new)

print (corpus)

#word = id2word[[0][:1][0]]
#print (word)<jupyter_output><empty_output><jupyter_text>**latent Drichlet Allocation** totally based on numerical Analysis<jupyter_code>lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                           id2word=id2word,
                                           num_topics=30,
                                           random_state=100,
                                           update_every=1,
                                           chunksize=100,
                                           passes=10,
                                           alpha="auto")<jupyter_output><empty_output><jupyter_text>**visualizing clusters**<jupyter_code>pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word, mds="mmds", R=30)
vis<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11303988427361025, 'fraction_numerical': 0.011985947509816079, 'mean_word_length': 4.0, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '169150', 'license': 'MIT', 'n_tokens_mistral': 1649, 'n_tokens_neox': 1589, 'n_words': 413, 'path': 'Reinforcement-learning-Model_Of_NOBEL_QURAN.ipynb', 'repo_name': 'MuhammadOsamaBinJafar/NOBEL_QURAN-and-Data_Science', 'size': '4839'}