<jupyter_start><jupyter_text>Credit: this file is created by <NAME>, as the Machine Learning developer in this project @https://github.com/soohmatthew/UROP_mat2vec  Import Data<jupyter_code>import json
import ast
import sys
import os
import pandas as pd
import nltk
import re

if not os.getcwd().endswith('UROP_mat2vec'):
    print('Please ensure that you are in the proper working directory')

from utils.process import MaterialsTextProcessor
text_processor = MaterialsTextProcessor()

with open('data/relevant_articles_material_science.json') as json_file:
    relevant = json.load(json_file)

relevant_articles = ast.literal_eval(relevant)
relevant_df = pd.DataFrame({'abstracts' : relevant_articles})

with open('data/irrelevant_articles.json') as json_file:
    irrelevant = json.load(json_file)
    
irrelevant_articles = [i for i in irrelevant if i != '']

for fname in os.listdir('data/'):
    if 'irrelevant_articles_' in fname:
        with open('data/' + fname) as json_file:
            irrelevant_articles += ast.literal_eval(json.load(json_file))
            
irrelevant_df = pd.DataFrame({'abstracts' : irrelevant_articles})<jupyter_output><empty_output><jupyter_text>Process Abstracts1. Remove foreign language2. Remove common words like "Abstracts"<jupyter_code>words = set(nltk.corpus.words.words())

def language_checker(string):
    remove_foreign = " ".join(w for w in nltk.wordpunct_tokenize(string) if w.lower() in words or not w.isalpha())
    # If more than 30% of words are cut, remove article, else keep
    if len(remove_foreign)/len(string) <= 0.5:

        return ""
    else:
        return string
    
def remove_weblinks(string):
    return re.sub('https?://[A-Za-z0-9./]+','', string)
    
relevant_df = relevant_df[relevant_df['abstracts'].apply(language_checker) != ""]
irrelevant_df = irrelevant_df[irrelevant_df['abstracts'].apply(language_checker) != ""]
relevant_df = relevant_df[relevant_df['abstracts'].apply(remove_weblinks) != ""]
irrelevant_df = irrelevant_df[irrelevant_df['abstracts'].apply(remove_weblinks) != ""]
irrelevant_df['abstracts'] = irrelevant_df['abstracts'].apply(lambda x: x.replace("Background", ""))
irrelevant_df['abstracts'] = irrelevant_df['abstracts'].apply(lambda x: x.replace("Abstract", ""))
relevant_df['abstracts'] = relevant_df['abstracts'].apply(lambda x: x.replace("Background", ""))
relevant_df['abstracts'] = relevant_df['abstracts'].apply(lambda x: x.replace("Abstract", ""))
relevant_df['label'] = 1
irrelevant_df['label'] = 0
df = pd.concat([relevant_df, irrelevant_df])<jupyter_output><empty_output><jupyter_text>Train Test split<jupyter_code>from sklearn.model_selection import train_test_split

x = df.abstracts
y = df.label

SEED = 2000
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3, random_state=SEED)
print(y_train[y_train == 1].shape)
print(y_train[y_train == 0].shape)

print(y_test[y_test == 1].shape)
print(y_test[y_test == 0].shape)<jupyter_output>(714,)
(3206,)
(304,)
(1376,)<jupyter_text>Pipeline: TF-IDF -> Classifier Trying out different classifiers to get the best results<jupyter_code>from sklearn.svm import LinearSVC, SVC
from sklearn.ensemble import AdaBoostClassifier
from sklearn.naive_bayes import MultinomialNB, BernoulliNB
from sklearn.linear_model import RidgeClassifier, LogisticRegression
from sklearn.neighbors import NearestCentroid
from sklearn.feature_selection import SelectFromModel
from sklearn.pipeline import Pipeline
from time import time
from sklearn.metrics import accuracy_score, f1_score
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
from xgboost import XGBClassifier

names = ["Logistic Regression", "Linear SVC", "LinearSVC with L1-based feature selection", "Multinomial NB", 
         "Bernoulli NB", "Ridge Classifier", "AdaBoost", "Nearest Centroid", "XGBoost"]
classifiers = [LogisticRegression(),
    LinearSVC(),
    Pipeline([('feature_selection', SelectFromModel(LinearSVC(penalty="l1", dual=False))), ('classification', LinearSVC(penalty="l2"))]),
    MultinomialNB(),
    BernoulliNB(),
    RidgeClassifier(),
    AdaBoostClassifier(),
    NearestCentroid(), 
    XGBClassifier(random_state=1,learning_rate=0.01)]

zipped_clf = zip(names,classifiers)

tvec = TfidfVectorizer()
def accuracy_summary(pipeline, x_train, y_train, x_test, y_test):
    if len(x_test[y_test == 0]) / (len(x_test)*1.) > 0.5:
        null_accuracy = len(x_test[y_test == 0]) / (len(x_test)*1.)
    else:
        null_accuracy = 1. - (len(x_test[y_test == 0]) / (len(x_test)*1.))
    t0 = time()
    model = pipeline.fit(x_train, y_train)
    y_pred = model.predict(x_test)
    train_test_time = time() - t0
    accuracy = accuracy_score(y_test, y_pred)
    f1 = f1_score(y_test, y_pred, labels = np.unique(y_pred))
    return accuracy, f1, train_test_time

def classifier_comparator(vectorizer=tvec, n_features=10000, stop_words=None, ngram_range=(1, 1), classifier=zipped_clf):
    result = []
    vectorizer.set_params(stop_words=stop_words, max_features=n_features, ngram_range=ngram_range)
    for n,c in classifier:
        checker_pipeline = Pipeline([('vectorizer', vectorizer),
                                     ('classifier', c)])
        clf_accuracy, f1_score, tt_time = accuracy_summary(checker_pipeline, x_train, y_train, x_test, y_test)
        result.append((n, clf_accuracy, f1_score, tt_time))
    return result

trigram_result = classifier_comparator(n_features=20000 ,ngram_range=(1,3))

trigram_result<jupyter_output>C:\Users\Dell\Anaconda3\lib\site-packages\sklearn\preprocessing\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.
  if diff:<jupyter_text>Pipeline: TF-IDF -> Logistic Regression<jupyter_code>pipeline = Pipeline([('vectorizer', TfidfVectorizer()), ('classifier', LogisticRegression())])
model = pipeline.fit(x_train, y_train)
y_pred = model.predict(x_test)
accuracy = accuracy_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)
f1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12469357738192516, 'fraction_numerical': 0.01323745710083347, 'mean_word_length': 5.616216216216216, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '302906', 'license': 'MIT', 'n_tokens_mistral': 2169, 'n_tokens_neox': 2061, 'n_words': 439, 'path': 'UROPcode/logistic_regression.ipynb', 'repo_name': 'YingjieQiao/WebScraping', 'size': '6125'}