<jupyter_start><jupyter_text>IMPLEMENTATION OF BASIC CONCEPTS OF NATURAL LANGUAGE PROCESSING USING SKLEARN 1. Importing Datasets<jupyter_code>from sklearn.datasets import fetch_openml
mice = fetch_openml(name='miceprotein', version=4)
mice.target<jupyter_output><empty_output><jupyter_text>2. Tokenization<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
sentence = ['sklearn is a library of machine learning',
           'This shows tokenization using sklearn']
vector = CountVectorizer()
X = vector.fit_transform(sentence)
print(vector.get_feature_names())
 print(X.toarray())<jupyter_output>[[1 1 1 1 1 0 1 0 0 0]
 [0 0 0 0 0 1 1 1 1 1]]<jupyter_text>3. Preprocessing<jupyter_code>from nltk.corpus import stopwords
from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS
ENGLISH_STOP_WORDS = set( stopwords.words('english') ).union( set(ENGLISH_STOP_WORDS) )
ENGLISH_STOP_WORDS
#REMOVING stop words from sentences

from nltk.corpus import stopwords 
from nltk.tokenize import word_tokenize 

data = 'Google LLC is an American multinational technology company that specializes in Internet-related services and products'
tokens = word_tokenize(data) 
after = [w for w in tokens if not w in ENGLISH_STOP_WORDS] 
after = []

for w in tokens: 
    if w not in ENGLISH_STOP_WORDS: 
        after.append(w) 
        


after<jupyter_output><empty_output><jupyter_text>SENTIMENT ANALYSIS USING SKLEARN<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer

data = []
data_labels = []
with open("P:/NLP/pos_tweets.txt") as f:
    for i in f: 
        data.append(i) 
        data_labels.append('pos')

with open("P:/NLP/neg_tweets.txt",encoding="utf8") as f:
    for i in f: 
        data.append(i)
        data_labels.append('neg')
vectorizer = CountVectorizer(
    analyzer = 'word',
    lowercase = False,
)
features = vectorizer.fit_transform(
    data
)
features_nd = features.toarray() # for easy usage
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test  = train_test_split(
        features_nd, 
        data_labels,
        train_size=0.80, 
        random_state=1234)
from sklearn.linear_model import LogisticRegression
log_model = LogisticRegression()
log_model = log_model.fit(X=X_train, y=y_train)
y_pred = log_model.predict(X_test)
import random
j = random.randint(0,len(X_test)-7)
for i in range(j,j+7):
    print(y_pred[0])
    ind = features_nd.tolist().index(X_test[i].tolist())
    print(data[ind].strip())
from sklearn.metrics import accuracy_score
print(accuracy_score(y_test, y_pred))<jupyter_output>0.800498753117207<jupyter_text>TF-IDF Vectorization<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
sent = ['Scikit Learn is one of the most advanced and powerful library of Machine Learning using Python. Scikit Learn gives us a practical access to a lot of Machine Learning Algorithms']
vectors = TfidfVectorizer()
Y = vectors.fit_transform(sent)
print(vectors.get_feature_names())
print(Y.shape)<jupyter_output>(1, 22)
{'fraction_non_alphanumeric': 0.09275647328744674, 'fraction_numerical': 0.01802687643395608, 'mean_word_length': 5.215885947046843, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '330859', 'license': 'MIT', 'n_tokens_mistral': 1106, 'n_tokens_neox': 1047, 'n_words': 256, 'path': 'B2-NLP/Prathamesh_NLP_ScikitLearn.ipynb', 'repo_name': 'Shreyansh-Gupta/Open-contributions', 'size': '3051'}