<jupyter_start><jupyter_text>Vector Representation of Text                                                                                                    <NAME> :)<jupyter_code>Article = ["""Perhaps one of the most significant advances made by Arabic mathematics began at this time with the work of al-Khwarizmi, namely
the beginnings of algebra. It is important to understand just how significant this new idea was. It was a revolutionary move away from
the Greek concept of mathematics which was essentially geometry. Algebra was a unifying theory which allowed rational
numbers, irrational numbers, geometrical magnitudes, etc., to all be treated as "algebraic objects". It gave mathematics a whole new
development path so much broader in concept to that which had existed before, and provided a vehicle for future development of the
subject. Another important aspect of the introduction of algebraic ideas was that it allowed mathematics to be applied to itself in a
way which had not happened before.""",

 """ربما كانت أحد أهم التطورات التي قامت بها الرياضيات العربية التي بدأت في هذا الوقت بعمل الخوارزمي  وهي بدايات الجبر،ومن المهم فهم كيف كانت هذه الفكرة الجديدة مهمة، فقد كانت خطوة ثورية بعيدا عن
المفهوم اليوناني للرياضيات التي هي في جوهرها  هندسة، الجبركان نظرية موحدة تتحيح الأعداد الكسرية و الأعداد اللا كسرية ، والمقادير الهندسية و غيرها ، أن تتعامل على أنها أجسام جبرية، و أعطت الرياضيات ككل مسارا جديدًا للتطوربمفهوم 
 أوسع بكثير من الذي كان موجودًا من قبل ، وقدم وسيلة للتنمية في هذا الموضوع مستقبلا .و جانب آخر مهم لإدخال أفكار الجبر و هو أنه سمح بتطبيق الرياضيات على نفسها 
بطريقة  لم تحدث من قبل."""
]

print(Article)
!pip install gensim<jupyter_output>Collecting gensim
  Downloading gensim-4.0.1-cp38-cp38-win_amd64.whl (23.9 MB)
Requirement already satisfied: smart-open>=1.8.1 in c:\users\asus\anaconda3\lib\site-packages (from gensim) (5.1.0)
Requirement already satisfied: scipy>=0.18.1 in c:\users\asus\anaconda3\lib\site-packages (from gensim) (1.6.2)
Requirement already satisfied: numpy>=1.11.3 in c:\users\asus\anaconda3\lib\site-packages (from gensim) (1.19.5)
Collecting Cython==0.29.21
  Downloading Cython-0.29.21-cp38-cp38-win_amd64.whl (1.7 MB)
Installing collected packages: Cython, gensim
  Attempting uninstall: Cython
    Found existing installation: Cython 0.29.23
    Uninstalling Cython-0.29.23:
      Successfully uninstalled Cython-0.29.23
Successfully installed Cython-0.29.21 gensim-4.0.1<jupyter_text>Pre-processing data<jupyter_code>import string
import nltk
import re
from nltk.corpus import stopwords
nltk.download('stopwords')
from nltk.tokenize import word_tokenize
nltk.download('punkt')
punctuations = '''`÷×؛<>_()*&^%][ـ،/:"؟.,'{}~¦+|!”…“–ـ''' + string.punctuation  #all punctuations 

def delete_punctuations(sentence):
  sentence = sentence.lower()
  my_clean_sentence = ''.join([item for item in sentence if item not in punctuations ])
  return my_clean_sentence
  
def sentence_tokenize(text):
  return word_tokenize(text)
  
tokenized_list = sentence_tokenize(delete_punctuations(Article[0]))
print(tokenized_list)<jupyter_output>['perhaps', 'one', 'of', 'the', 'most', 'significant', 'advances', 'made', 'by', 'arabic', 'mathematics', 'began', 'at', 'this', 'time', 'with', 'the', 'work', 'of', 'alkhwarizmi', 'namely', 'the', 'beginnings', 'of', 'algebra', 'it', 'is', 'important', 'to', 'understand', 'just', 'how', 'significant', 'this', 'new', 'idea', 'was', 'it', 'was', 'a', 'revolutionary', 'move', 'away', 'from', 'the', 'greek', 'concept', 'of', 'mathematics', 'which', 'was', 'essentially', 'geometry', 'algebra', 'was', 'a', 'unifying', 'theory', 'which', 'allowed', 'rational', 'numbers', 'irrational', 'numbers', 'geometrical', 'magnitudes', 'etc', 'to', 'all', 'be', 'treated', 'as', 'algebraic', 'objects', 'it', 'gave', 'mathematics', 'a', 'whole', 'new', 'development', 'path', 'so', 'much', 'broader', 'in', 'concept', 'to', 'that', 'which', 'had', 'existed', 'before', 'and', 'provided', 'a', 'vehicle', 'for', 'future', 'development', 'of', 'the', 'subject', 'another', 'important', 'aspect', 'of', 'the', 'in[...]<jupyter_text>CountVectorizer and Dictionary of Word Frequency<jupyter_code>import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
  
 
count_vec = CountVectorizer(ngram_range=(1,1),stop_words='english')
#Transform
count_data = count_vec.fit_transform([Article[0]])
print(count_vec.get_feature_names())
#Create dataframe
my_df=pd.DataFrame(count_data.toarray(),columns=count_vec.get_feature_names()).T
my_df.head(10)<jupyter_output>['advances', 'al', 'algebra', 'algebraic', 'allowed', 'applied', 'arabic', 'aspect', 'away', 'began', 'beginnings', 'broader', 'concept', 'development', 'essentially', 'existed', 'future', 'gave', 'geometrical', 'geometry', 'greek', 'happened', 'idea', 'ideas', 'important', 'introduction', 'irrational', 'just', 'khwarizmi', 'magnitudes', 'mathematics', 'new', 'numbers', 'objects', 'path', 'provided', 'rational', 'revolutionary', 'significant', 'subject', 'theory', 'time', 'treated', 'understand', 'unifying', 'vehicle', 'way', 'work']<jupyter_text>Arabic text<jupyter_code>count_vec=CountVectorizer(token_pattern=r"(?u)\b\w\w+\b")
arr = count_vec.fit_transform([Article[1]]).toarray()
print(arr)
print('\nvocabulary list:\n')
for key,value in count_vec.vocabulary_.items():
    print(key,value)
print(count_data)<jupyter_output>(0, 38)	2
  (0, 0)	1
  (0, 6)	1
  (0, 30)	4
  (0, 9)	1
  (0, 41)	1
  (0, 47)	1
  (0, 1)	1
  (0, 28)	1
  (0, 10)	1
  (0, 2)	2
  (0, 24)	2
  (0, 43)	1
  (0, 27)	1
  (0, 31)	2
  (0, 22)	1
  (0, 37)	1
  (0, 8)	1
  (0, 20)	1
  (0, 12)	2
  (0, 14)	1
  (0, 19)	1
  (0, 44)	1
  (0, 40)	1
  (0, 4)	2
  (0, 36)	1
  (0, 32)	2
  (0, 26)	1
  (0, 18)	1
  (0, 29)	1
  (0, 42)	1
  (0, 3)	2
  (0, 33)	1
  (0, 17)	1
  (0, 13)	2
  (0, 34)	1
  (0, 11)	1
  (0, 15)	1
  (0, 35)	1
  (0, 45)	1
  (0, 16)	1
  (0, 39)	1
  (0, 7)	1
  (0, 25)	1
  (0, 23)	1
  (0, 5)	1
  (0, 46)	1
  (0, 21)	1<jupyter_text>TF-IDF : - Term Frequency / Inverse Document Frequency !<jupyter_code>import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

 
#Define tf-idf
tf_idf_vec = TfidfVectorizer(use_idf=True, 
                        smooth_idf=False,  
                        ngram_range=(1,1),stop_words='english')
#Transform
tf_idf_data = tf_idf_vec.fit_transform([Article[0]])
 
#Create dataframe
tf_idf_dataframe=pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names()).T
tf_idf_dataframe.head(10)<jupyter_output><empty_output><jupyter_text>CBoW Bag of Words Model 'Keras'<jupyter_code>from keras.preprocessing.text import Tokenizer

model = Tokenizer() #init tokenizer
model.fit_on_texts(Article[0]) #Change this Article[0] to 1 if you want insert arabic text
print(f'Key : {list(model.word_index.keys())}') #Print Keys 



#Summarize what was learned:
print(model.word_counts)
print(model.document_count)
print(model.word_index)
print(model.word_docs)


#Create bag of words representation :
text_to_vector = model.texts_to_matrix(Article[0], mode='count')
my_cbow_result = pd.DataFrame(text_to_vector)
my_cbow_result<jupyter_output>Key : ['e', 'a', 't', 'i', 'o', 'n', 'h', 'r', 's', 'c', 'm', 'l', 'd', 'w', 'b', 'f', 'p', 'g', 'u', 'y', 'v', 'k', 'j', 'z', 'x']
OrderedDict([('p', 15), ('e', 77), ('r', 33), ('h', 35), ('a', 72), ('s', 33), ('o', 45), ('n', 38), ('f', 16), ('t', 67), ('m', 25), ('i', 52), ('g', 15), ('c', 26), ('d', 22), ('v', 8), ('b', 17), ('y', 9), ('w', 20), ('k', 3), ('l', 24), ('z', 1), ('u', 12), ('j', 3), ('x', 1)])
814
{'e': 1, 'a': 2, 't': 3, 'i': 4, 'o': 5, 'n': 6, 'h': 7, 'r': 8, 's': 9, 'c': 10, 'm': 11, 'l': 12, 'd': 13, 'w': 14, 'b': 15, 'f': 16, 'p': 17, 'g': 18, 'u': 19, 'y': 20, 'v': 21, 'k': 22, 'j': 23, 'z': 24, 'x': 25}
defaultdict(<class 'int'>, {'p': 15, 'e': 77, 'r': 33, 'h': 35, 'a': 72, 's': 33, 'o': 45, 'n': 38, 'f': 16, 't': 67, 'm': 25, 'i': 52, 'g': 15, 'c': 26, 'd': 22, 'v': 8, 'b': 17, 'y': 9, 'w': 20, 'k': 3, 'l': 24, 'z': 1, 'u': 12, 'j': 3, 'x': 1})<jupyter_text>OneHotEncoder (Sklearn,Keras)<jupyter_code>#From SKLEARN :

import numpy as np
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder

values = np.array(tokenized_list)
print(values)

#Integer encode
label_encoder = LabelEncoder()
my_int_encoded = label_encoder.fit_transform(values)
print(my_int_encoded)

#Binary encode
onehot_encoder = OneHotEncoder(sparse=False)
my_int_encoded = my_int_encoded.reshape(len(my_int_encoded), 1)
onehot_encoded_values = onehot_encoder.fit_transform(my_int_encoded)
print(onehot_encoded_values)
inverted_values = label_encoder.inverse_transform([np.argmax(onehot_encoded_values[0, :])])
print(inverted_values)
#From Keras :

from keras.preprocessing.text import one_hot
from keras.preprocessing.text import text_to_word_sequence

#Define the document: 
words = set(text_to_word_sequence(Article[0]))
vocab_size = len(words)
print(vocab_size)

#Integer encode the document:
result = one_hot(Article[0], round(vocab_size))
print(result)<jupyter_output>84
[1, 79, 57, 6, 59, 40, 72, 82, 83, 65, 46, 79, 8, 46, 30, 18, 6, 57, 57, 9, 22, 16, 6, 5, 57, 8, 71, 23, 65, 37, 52, 55, 21, 40, 46, 62, 81, 67, 71, 67, 3, 27, 38, 83, 42, 6, 59, 22, 57, 46, 49, 67, 5, 54, 8, 67, 3, 29, 1, 49, 32, 23, 9, 57, 9, 26, 67, 19, 37, 17, 59, 69, 83, 15, 23, 71, 49, 46, 3, 46, 62, 9, 32, 7, 82, 53, 41, 22, 37, 74, 49, 80, 14, 60, 40, 31, 3, 77, 48, 65, 9, 57, 6, 75, 28, 65, 41, 57, 6, 39, 57, 15, 51, 67, 74, 71, 32, 46, 37, 59, 11, 37, 44, 41, 3, 41, 49, 80, 20, 58, 60]<jupyter_text>WORD2VEC<jupyter_code>from gensim.models import Word2Vec,keyedvectors

model = Word2Vec([tokenized_list] , min_count=1, size=32)
model.most_similar("important")

model.similarity("important", "mathematics") #Test similarity between two words

model.similar_by_word("one")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18167104996965405, 'fraction_numerical': 0.06645761683188348, 'mean_word_length': 4.995755003032141, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '157610', 'license': 'Apache-2.0', 'n_tokens_mistral': 4744, 'n_tokens_neox': 4017, 'n_words': 1146, 'path': 'NLP - TP2.ipynb', 'repo_name': 'Fadwaaaaa/NLP.TP2', 'size': '9893'}