<jupyter_start><jupyter_text>Libs<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
import pandas
from sklearn.decomposition import TruncatedSVD
from sklearn.preprocessing import StandardScaler
from google.colab import drive
import ast
import re
import pickle<jupyter_output><empty_output><jupyter_text>1 - Collecting Data<jupyter_code>data = pandas.read_csv('../Data/preprocessed.csv')
print(data['body'])
freq_dist = pandas.read_csv('../Data/FreqDist_sorted.csv',index_col=False)
freq_dist.head()<jupyter_output><empty_output><jupyter_text>TF-ICF words<jupyter_code># most of words with frequency of 1, are noises or bad detected
freq_dist_slice = freq_dist.sort_values(by='freq',ascending=False)[freq_dist['freq']>2]
print(len(freq_dist_slice))
freq_dist_slice
# based on tf_icf we may need only 20% of words
'''
tf_icf_words = freq_dist_slice.iloc[:int(len(freq_dist_slice)*.20)]['word'].values
other_words = freq_dist_slice.iloc[int(len(freq_dist_slice)*.20):]['word'].values
print(len(tf_icf_words))
tf_icf_words[:10]
'''<jupyter_output><empty_output><jupyter_text>2 - Extract Features  TF-IDF words<jupyter_code>max_features = int(freq_dist_slice.shape[0]*0.1)
max_features
vectorizer = TfidfVectorizer(analyzer='word', max_features=max_features)
X = vectorizer.fit_transform(data['body'])
print(type(X))
# to save
pickle.dump(vectorizer,open("../Data/tf_idf.pkl","wb"))

# to load 
# tf = pickle.load(open("tfidf1.pkl", 'rb'))
# tf_new = TfidfVectorizer(analyzer='word', ngram_range=(1,2), stop_words = "english", lowercase = True,
#                           max_features = 500000, vocabulary = tf.vocabulary_)
# feature_names = vectorizer.get_feature_names()
X<jupyter_output><empty_output><jupyter_text>second way<jupyter_code>'''
max_features = int(freq_dist_slice.shape[0]*0.002)
max_features
'''
'''
from sklearn.feature_selection import SelectKBest
vectorizer = TfidfVectorizer(analyzer='word')
df_t = vectorizer.fit_transform(data['body'])
'''
'''
df_t_reduced = SelectKBest(k=max_features).fit_transform(df_t, data['tag'])
max_features
'''
'''
df_t_reduced.shape
'''<jupyter_output><empty_output><jupyter_text>SVD<jupyter_code>svd_features = int(max_features*0.05)
svd_features
svd = TruncatedSVD(n_components=svd_features, random_state=42)
features = svd.fit_transform(X)
print(features.shape) 
# to save
pickle.dump(svd, open("../Data/svd.p","wb"))

# To load again
# svd = pickle.load(open('../Data/svd.p', 'r'))<jupyter_output><empty_output><jupyter_text>Saving Data<jupyter_code>df = pandas.DataFrame(features)
df.to_csv('../Data/features.csv', index=False)
df.shape
df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14711429649188984, 'fraction_numerical': 0.012825348924933987, 'mean_word_length': 8.240418118466899, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '248222', 'license': 'MIT', 'n_tokens_mistral': 1029, 'n_tokens_neox': 1002, 'n_words': 141, 'path': 'Spider/ipython(guide)/step_by_step_2_Vectorizing.ipynb', 'repo_name': 'ZeinabTaghavi/text_multi_label_classification', 'size': '2651'}