<jupyter_start><jupyter_text>Data Analysis<jupyter_code>data = pd.read_csv("train.csv",engine = 'python')
data.tail()
colum = data.columns
for i in colum:
    print(f'{len(set(data[i]))} different values in the {i} column')
print(f"\ntotal number of examples {len(data)}")

#Host, link, Time(ET), Time(GMT),is of no use for trainig the function
data = data.drop(["Host", "Link", "Date(ET)", "Time(ET)", "time(GMT)"], axis=1)
colum = data.columns
for i in colum:
    print(f'{len(set(data[i]))} different values in the {i} column')
print(f"\ntotal number of examples {len(data)}")


list(set(data["Source"])) # differnet values in "Source" column
# repalcing FACEBOOK to Facebook 
data.replace(to_replace='FACEBOOK', value='Facebook',inplace=True)
# Now there are only 4 different values in "Source" column
Counter(data.loc[:,"Source"]) #  # distribution of differnet values in "Source" column
Counter(data.iloc[:,[-1]]['Patient_Tag']) # distribution of labels in the "Patien_Tag" column
# It's an unbalanced data
dummy = {}
for i in list(set(data["Source"])):
    print(i,"---", Counter(data.iloc[:,[0,-1]][data['Source'] == i]['Patient_Tag']))
# distribution of labels with reference to each values in "Source" column
replace_ = {}
for index, i in enumerate(list(set(data["Source"])),start=1):
    replace_[index] = i
    data.replace(to_replace=i, value=index,inplace=True)
data.fillna('UNK',inplace=True)
list(set(data["Source"]))
data.fillna('UNK',inplace=True)<jupyter_output><empty_output><jupyter_text>Vocab creation<jupyter_code>import re
rep_with = ['.', '?', '/', '\n', '(', ')','[', ']', '{', '}', '-','"','!', '|' ]
def rep_(sent):
    for i in rep_with:
        sent = sent.replace(i,' ').replace('$', ' ').replace(',','').replace("'",'')
    return sent
import re
import num2words

def n2w(text):
    return re.sub(r"(\d+)", lambda x: num2words.num2words(int(x.group(0))), text)
def preprocess(data,pos):
    sent = []
    for i in range(len(data)):
        try:sent.append(n2w(rep_(data.iloc[i,pos])))
        except:print(data.iloc[i,pos])
    return sent
sent = preprocess(data, 2)
sent[0]
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
vectorizer = TfidfVectorizer()
vectorizer.fit(sent)
# summarize
print("length of vocabulry --",len(vectorizer.vocabulary_))
# print(vectorizer_1.vocabulary_)
# print(vectorizer_2.idf_)
vector = vectorizer.transform(sent)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>x = np.array(vector.toarray()) # text
m = np.array(data['Source']).reshape(1157,1) # source
x = np.concatenate((m,x),axis=1) # source + text
y = np.array(data.loc[:,'Patient_Tag']) # label
x.shape, y.shape
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size=0.2)
from sklearn.linear_model import LogisticRegression
model = LogisticRegression(class_weight='balanced', C=1.5)

model.fit(x_train, y_train)
predictions = model.predict(x_test)
score = model.score(x_test,y_test)
score
from sklearn import svm
model = svm.SVC(class_weight='balanced', C=1.5)
model.fit(x_train, y_train,)

predictions = model.predict(x_test)
score = model.score(x_test,y_test)
score
import xgboost as xgb
data_dmatrix = xgb.DMatrix(data=x_train,label=y_train)
xg_reg = xgb.XGBClassifier(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.05,
                max_depth = 100, alpha = 0.75, n_estimators = 100, n_jobs = 32)
xg_reg.fit(x_train,y_train)
predictions = xg_reg.predict(x_test)
score = xg_reg.score(x_test,y_test)
score
params = {"objective":"reg:linear",'colsample_bytree': 0.3,'learning_rate': 0.1,
                'max_depth': 5, 'alpha': 10}

cv_results = xgb.cv(dtrain=data_dmatrix, params=params, nfold=3,num_boost_round=50,
                 early_stopping_rounds=10,metrics="merror", as_pandas=True, seed=123)
cv_results<jupyter_output><empty_output><jupyter_text>Results analysis<jupyter_code>import seaborn as sns
from sklearn import metrics
import matplotlib.pyplot as plt
cm = metrics.confusion_matrix(y_test, predictions)
plt.figure(figsize=(9,9))
sns.heatmap(cm, annot=True, fmt=".3f", linewidths=.5, square = True, cmap = 'Blues_r');
plt.ylabel('Actual label');
plt.xlabel('Predicted label');
all_sample_title = 'Accuracy Score: {0}'.format(score)
plt.title(all_sample_title, size = 15);<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code>data_ = pd.read_csv("test.csv",engine='python')
data_ = data_.drop(["Host", "Link", "Date(ET)", "Time(ET)", "time(GMT)", "Unnamed: 9", "Index"], axis=1)
data_.replace(to_replace='FACEBOOK', value='Facebook',inplace=True)
replace_ = {}
for index, i in enumerate(list(set(data_["Source"])),start=1):
    replace_[index] = i
    data_.replace(to_replace=i, value=index,inplace=True)
data_.fillna('UNK',inplace=True)
sent = preprocess(data_, 2)
data_.head()
vector = vectorizer.transform(sent)
x = np.array(vector.toarray())
m = np.array(data_['Source']).reshape(571,1) # source
x = np.concatenate((m,x),axis=1) # source + text
# predictions = model.predict(x)
predictions = xg_reg.predict(x)
predictions.shape<jupyter_output><empty_output><jupyter_text>Saving<jupyter_code>format = 'Index,Patient_Tag\n'
for index, i in enumerate(predictions,start=1):
    format+=str(index)+','+str(i) + '\n'
with open("submission_7.csv", "w") as f:
    f.write(format.strip())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17411459299091414, 'fraction_numerical': 0.014277767476358243, 'mean_word_length': 6.470914127423823, 'pattern_counts': {'":': 1, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '569745', 'license': 'Apache-2.0', 'n_tokens_mistral': 2058, 'n_tokens_neox': 2000, 'n_words': 381, 'path': 'train_test_save_basic.ipynb', 'repo_name': 'raotnameh/text-classification', 'size': '5393'}