<jupyter_start><jupyter_text>Nama : <NAME>   NPM : 19312039   Kelas : IF Gab 1  **IMPORT LIBRARY YANG PENTING**<jupyter_code>import nltk
nltk.download('stopwords')
nltk.download('wordnet')
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer 
from nltk.stem.snowball import SnowballStemmer
from nltk.stem.wordnet import WordNetLemmatizer
import pandas as pd
import re
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!<jupyter_text>** Contoh Sentence/ Kalimat yang akan di Training atau process**<jupyter_code>sentences = ["We are reading about Natural Language Processing Here",
            "Natural Language Processing making computers comprehend language data",
            "The field of Natural Language Processing is evolving everyday"]<jupyter_output><empty_output><jupyter_text>** Memasukan Sentence ke dalam object list menggunakan library pandas**<jupyter_code>corpus = pd.Series(sentences)
corpus<jupyter_output><empty_output><jupyter_text>** Membuat Fungsi untuk alfabet, number & beberapa kata khusus**  input adalah corpus diatas  outputnya adalah text corpus yang sudah di bersihkan<jupyter_code>def text_clean(corpus, keep_list):

    cleaned_corpus = pd.Series()                                                 # ini Series kosong untuk menampung corups output bersih yang dihasilkan
    for row in corpus:
        qs = []
        for word in row.split():
            if word not in keep_list:                                           #Keep list adalah list bawaan dari library nltk
                p1 = re.sub(pattern='[^a-zA-Z0-9]',repl=' ',string=word)        # menggunakan regex untuk mengambil pola alfabet dan numbers saja
                p1 = p1.lower()                                                 # menggubah ke dalam no captital
                qs.append(p1)                                                   # memasukan p1 ke array qs
            else : qs.append(word)
        cleaned_corpus = cleaned_corpus.append(pd.Series(' '.join(qs)))
    return cleaned_corpus<jupyter_output><empty_output><jupyter_text>** membuat lemmatizer**<jupyter_code>def lemmatize(corpus):
    lem = WordNetLemmatizer()                                                   # WordNetLemmatizer bawaan NLTK
    corpus = [[lem.lemmatize(x, pos = 'v') for x in x] for x in corpus]         # me-lemmitizer corpus
    return corpus<jupyter_output><empty_output><jupyter_text>** membuat Steam function**<jupyter_code>def stem(corpus, stem_type = None):
    if stem_type == 'snowball':
        stemmer = SnowballStemmer(language = 'english')
        corpus = [[stemmer.stem(x) for x in x] for x in corpus]
    else :
        stemmer = PorterStemmer()
        corpus = [[stemmer.stem(x) for x in x] for x in corpus]
    return corpus<jupyter_output><empty_output><jupyter_text>** membuat/ meggunakan stop_words**<jupyter_code>def stopwords_removal(corpus):
    wh_words = ['who', 'what', 'when', 'why', 'how', 'which', 'where', 'whom']
    stop = set(stopwords.words('english'))
    for word in wh_words:
        stop.remove(word)
    corpus = [[x for x in x.split() if x not in stop] for x in corpus]
    return corpus<jupyter_output><empty_output><jupyter_text>** membuat fungsi preprocessing**<jupyter_code>def preprocess(corpus, keep_list, cleaning = True, stemming = False, stem_type = None, lemmatization = False, remove_stopwords = True):
    '''
    Purpose : Function to perform all pre-processing tasks (cleaning, stemming, lemmatization, stopwords removal etc.)
    
    Input : 
    'corpus' - Text corpus on which pre-processing tasks will be performed
    'keep_list' - List of words to be retained during cleaning process
    'cleaning', 'stemming', 'lemmatization', 'remove_stopwords' - Boolean variables indicating whether a particular task should 
                                                                  be performed or not
    'stem_type' - Choose between Porter stemmer or Snowball(Porter2) stemmer. Default is "None", which corresponds to Porter
                  Stemmer. 'snowball' corresponds to Snowball Stemmer
    
    Note : Either stemming or lemmatization should be used. There's no benefit of using both of them together
    
    Output : Returns the processed text corpus
    
    '''
    print(f'ini keep list {keep_list}')
    
    if cleaning == True:
        corpus = text_clean(corpus, keep_list)
    
    if remove_stopwords == True:
        corpus = stopwords_removal(corpus)
    else :
        corpus = [[x for x in x.split()] for x in corpus]
    
    if lemmatization == True:
        corpus = lemmatize(corpus)
        
        
    if stemming == True:
        corpus = stem(corpus, stem_type)
    
    corpus = [' '.join(x) for x in corpus]        

    return corpus
preprocessed_corpus = preprocess(corpus, keep_list = [], stemming = False, stem_type = None,
                                lemmatization = True, remove_stopwords = True)<jupyter_output>ini keep list []<jupyter_text>** Contoh Output Preprocessed_Corpus**<jupyter_code>print(f'Ini Adalah Output preprocessed_corpus -->  {preprocessed_corpus} \n')<jupyter_output>Ini Adalah Output preprocessed_corpus -->  ['read natural language process', 'natural language process make computers comprehend language data', 'field natural language process evolve everyday']<jupyter_text>** Vectorize**  Menghitung kata kata yang divektorisasi ** Mendfinisi Vectorize & bow_matrix**<jupyter_code>vectorizer = CountVectorizer()
bow_matrix = vectorizer.fit_transform(preprocessed_corpus)
print('Kata Kata yang sering muncul dalam sentence / Corpus \n')
print(vectorizer.get_feature_names())
print('\n Konvert dalam bentuk Matrix \n')
print(bow_matrix.toarray())
print(bow_matrix.toarray().shape)

vectorizer_ngram_range = CountVectorizer(analyzer='word', ngram_range=(1,3))
bow_matrix_ngram = vectorizer_ngram_range.fit_transform(preprocessed_corpus)

print(vectorizer_ngram_range.get_feature_names())
print(bow_matrix_ngram.toarray())

vectorizer_max_features = CountVectorizer(analyzer='word', ngram_range=(1,3), max_features = 6)
bow_matrix_max_features = vectorizer_max_features.fit_transform(preprocessed_corpus)
print(vectorizer_max_features.get_feature_names())
print(bow_matrix_max_features.toarray())
vectorizer_max_features = CountVectorizer(analyzer='word', ngram_range=(1,3), max_df = 3, min_df = 2)
bow_matrix_max_features = vectorizer_max_features.fit_transform(preprocessed_corpus)
print(vectorizer_max_features.get_feature_names())
print(bow_matrix_max_features.toarray())<jupyter_output>['language', 'language process', 'natural', 'natural language', 'natural language process', 'process']
[[1 1 1 1 1 1]
 [2 1 1 1 1 1]
 [1 1 1 1 1 1]]
{'fraction_non_alphanumeric': 0.09373202990224266, 'fraction_numerical': 0.0063254744105807935, 'mean_word_length': 3.5411227154046996, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '209072', 'license': 'Apache-2.0', 'n_tokens_mistral': 2252, 'n_tokens_neox': 2144, 'n_words': 580, 'path': 'Bagus_Bramantyo_Tugas.ipynb', 'repo_name': '3ndung/tutorials', 'size': '6965'}