<jupyter_start><jupyter_text>Introduction Implementation of Ruleex ANN-DT for sentiment analysis for English Language. The dataset used is Large Movie Review Dataset (Imdb reviews) which have total of 50K reviews having equal distribution in training and test dataset (25K each). Furthermore, the distribution of positive and negative reviews in training and test dataset is also equal (12.5K pos and 12.5 neg reviews in each training and test dataset). Here we take the reduced data for demonstration since the alogorithm is very slow. The model in use is simple deep neural network with tf-idf features. Dataset location: https://ai.stanford.edu/~amaas/data/sentiment/ ANN-DT Paper: https://ieeexplore.ieee.org/document/809084 Code reference: https://github.com/fantamat/ruleex For this notebook, use forked library: https://github.com/rohancode/ruleex_modified<jupyter_code>import warnings
warnings.filterwarnings('ignore')

from glob import glob
import os
import sys

import pandas as pd
import numpy as np
import re

import keras
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Flatten
from keras.layers.core import Dense

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split

from gtrain import FCNet

import tensorflow as tf
keras.__version__
tf.__version__<jupyter_output><empty_output><jupyter_text>Path of the directory where repository (https://github.com/rohancode/ruleex_modified) is cloned<jupyter_code>sys.path.append('/Users/Rohan/Desktop/Cl/XAI/Rulex/workspace')
from ruleex_modified.ruleex.deepred.model import DeepRedFCNet
import ruleex_modified.ruleex.anndt as ndt
import ruleex_modified.ruleex.deepred as dr
from tqdm import tqdm<jupyter_output><empty_output><jupyter_text>Sentiment Analysis Model<jupyter_code>def parse_folder(name):
    data = []
    for verdict in ('neg', 'pos'):
        for file in tqdm(glob(os.path.join(name, verdict, '*.txt'))):
            data.append({
                'text': open(file, encoding='utf8').read(),
                'verdict': verdict == 'pos'
            })
    return pd.DataFrame(data)

df_train = parse_folder('/Users/Rohan/Desktop/Cl/XAI/aclImdb/train/')
df_test = parse_folder('/Users/Rohan/Desktop/Cl/XAI/aclImdb/test/')

df = pd.concat([df_train, df_test])
df.reset_index(inplace=True)
df.drop(['index'], axis=1, inplace=True)
df = df.sample(frac=1)
df.verdict.value_counts()<jupyter_output><empty_output><jupyter_text>Working on reduced data for to reduce runtime for quick demonstration<jupyter_code>df = df.head(20000)
df.verdict.value_counts()
TAG_RE = re.compile(r'<[^>]+>')
def remove_tags(text):
    return TAG_RE.sub('', text)

def preprocess_text(sen):
    sentence = remove_tags(sen)
    sentence = re.sub('[^a-zA-Z]', ' ', sentence)
    sentence = re.sub(r"\s+[a-zA-Z]\s+", ' ', sentence)
    sentence = re.sub(r'\s+', ' ', sentence)
    return sentence

X_text = []
sentences = list(df['text'])
for sen in sentences:
    X_text.append(preprocess_text(sen))
    
y = df['verdict']
y = np.array(list(map(lambda x: [0,1] if x==True else [1,0], y)))
x_text, x_text_test, y, y_test = train_test_split(X_text, y, test_size=0.20, random_state=42)
vectorizer = TfidfVectorizer(max_features=500)
X_tfidf = vectorizer.fit_transform(x_text)
X_test_tfidf = vectorizer.transform(x_text_test)
model_mn = Sequential([
  Dense(100, activation='relu', input_shape=(500,)),
  Dense(30, activation='relu'),
  Dense(2, activation='softmax'),
])
model_mn.compile(
  optimizer='adam',
  loss='categorical_crossentropy',
  metrics=['accuracy'],
)
model_mn.fit(
  X_tfidf,
  y,
  epochs=3,
  batch_size=32,
)
model_mn.evaluate(X_test_tfidf, y_test)<jupyter_output>4000/4000 [==============================] - 0s 47us/step<jupyter_text>Ruleex ANN-DT<jupyter_code>directory = 'sentiment_new123'
data = 'imdb'
method = 'anndt'
name = data + "_" + method
tat_params = dict()
tat_params["save_dir"] = os.path.join("runs", directory, name)
if not os.path.isdir(tat_params["save_dir"]):
    os.makedirs(tat_params["save_dir"])
tat_params["init_restrictions"] = np.array([[0, 1] for _ in range(500)])
tat_params["act_val_num"] = 0
def TaT_anndt(tr_in_dict, tr_out_dict, tst_in_dict, tst_out_dict, params):
    
    layer_sizes = []
    layer_sizes.append(500)
    for layer in model_mn.layers:
        layer_sizes.append(layer.output_shape[-1])
    
    weights = model_mn.get_weights()
    
    net = DeepRedFCNet(layer_sizes)
    net.init_eval_weights(weights)
    
    p = {
        "varbose": 1,
        ndt.ATTRIBUTE_SELECTION: False,
    }

    if "min_train_samples" in params:
        p[ndt.MIN_TRAIN_SAMPLES] = params["min_train_samples"]

    if "max_depth" not in params:
        p[ndt.MAX_DEPTH] = 10
    else:
        p[ndt.MAX_DEPTH] = params["max_depth"]

    if "min_samples" in params:
        p[ndt.MIN_SAMPLES] = params["min_samples"]

    if "min_split_fraction" in params:
        p[ndt.MIN_SPLIT_FRACTION] = params["min_split_fraction"]

    stat_test = None
    if "split_test" in params:
        if params["split_test"]=="t":
            stat_test = ndt.test_t
        elif params["split_test"] == "welch":
            stat_test = ndt.test_welch
        elif params["split_test"] == "chi2":
            stat_test = ndt.test_chi2
        elif params["split_test"] == "f":
            stat_test = ndt.test_F
        else:
            stat_test = ndt.test_chi2
    else:
        stat_test = ndt.test_chi2

    if "measure" in params:
        if params["measure"]=="gini":
            measure = ndt.GiniMeasure
        elif params["measure"]=="missclass":
            measure = ndt.FidelityGain
        elif params["measure"]=="maxdiff":
            measure = ndt.MaxDifference
        elif params["measure"]=="var":
            measure = ndt.VarianceMeasure
        else:
            measure = ndt.EntropyMeasure
    else:
        measure = ndt.EntropyMeasure

    if "attr_selection" in params:
        if params["attr_selection"]=="absvar":
            p[ndt.ATTRIBUTE_SELECTION] = ndt.MODE_ABSOLUTE_VARIATION
        elif params["attr_selection"]=="missclass":
            p[ndt.ATTRIBUTE_SELECTION] = ndt.MODE_MISSCLASSIFICATION
        elif params["attr_selection"]=="conmissclass":
            p[ndt.ATTRIBUTE_SELECTION] = ndt.MODE_CONTINUOUS_MISSCLASSIFICATION

    if "measure_weights" in params:
        if params["measure_weights"]=="train":
            p[ndt.MEASURE_WEIGHTS] = ndt.MODE_TRAIN
        elif params["measure_weights"]=="all":
            p[ndt.MEASURE_WEIGHTS] = ndt.MODE_ALL
        else:
            p[ndt.MEASURE_WEIGHTS] = ndt.MODE_NONE

    if "force_sampling" in params:
        p[ndt.FORCE_SAMPLING] = params["force_sampling"]

    if "num_samples" in params:
        indexes = np.random.permutation(len(tr_in_dict["x"]))
        x_train = tr_in_dict["x"][indexes[:params["num_samples"]]]
    else:
        x_train = tr_in_dict["x"]

    if "vs_others" in params:
        model = lambda x: net.eval_binary_class(x, params["vs_others"])
    else:
        model = net.eval

    if "init_restrictions" in params:
        res = params["init_restrictions"]
    else:
        res = None

    p[ndt.SPLIT_TEST_AFTER] = 3
    rt = ndt.anndt(model, x_train, p,
                   stat_test=stat_test,
                   MeasureClass=measure,
                   init_restrictions=res,
                   sampler=ndt.BerNormalSampler(x_train, always_positive=True, sigma=0.01))
    rt.save(os.path.join(params["save_dir"], "rt.pic"))

    dt = DecisionTreeClassifier(max_depth=p["max_depth"])
    dt.fit(x_train, np.argmax(model(x_train), axis=1))
    dt = dr.sklearndt_to_ruletree(dt, one_class_on_leafs=True)
    print("rt.view_graph()")
    rt.view_graph(filename='mnist_tree.pdf', varbose=True)
    dt.save(os.path.join(params["save_dir"], "sklear_dt.pic"))
    inf = p[ndt.INF]
    rt_train = rt.eval_all(tr_in_dict["x"])
    dt_train = dt.eval_all(tr_in_dict["x"])
    ln = np.argmax(model(tr_in_dict["x"]), axis=1)
    inf["fidelity"] = np.mean(rt_train == ln)
    inf["dt_fidelity"] = np.mean(dt_train == ln)
    rt_val = rt.eval_all(tst_in_dict["x"])
    dt_val = dt.eval_all(tst_in_dict["x"])
    l = np.argmax(tst_out_dict["y"], axis=1)
    ln = np.argmax(model(tst_in_dict["x"]), axis=1)
    inf["val_fidelity"] = np.mean(ln == rt_val)
    inf["val_accuracy"] = np.mean(l == rt_val)
    inf["dt_val_fidelity"] = np.mean(ln == dt_val)
    inf["dt_val_accuracy"] = np.mean(l == dt_val)
    print("Validation {}: fidelity {}, val_fidelity {}, val_accuracy {}".format(params["act_val_num"], inf["fidelity"],
                                                                                inf["val_fidelity"],
                                                                                inf["val_accuracy"]))
    print("Validation {}: fidelity {}, val_fidelity {}, val_accuracy {}".format(params["act_val_num"], inf["dt_fidelity"],
                                                                                inf["dt_val_fidelity"],
                                                                                inf["dt_val_accuracy"]))
    return inf
x_train_final = X_tfidf.toarray()
x_test_final = X_test_tfidf.toarray()
inf = TaT_anndt({"x": x_train_final}, {"y": y}, {"x": x_test_final}, {"y": y_test}, tat_params)<jupyter_output>[anndt]: Generated new node with split x_37 > 0.019214667040972223 in train samples separation (3797, 12203)
[anndt]: Generated new node with split x_175 > 0.052310866126188585 in train samples separation (561, 3236)
[anndt]: Generated new node with split x_487 > 0.01358462706339282 in train samples separation (64, 497)
[anndt]: Stopping rule - fraction of the founded node is to low so the leaf is generated.
[anndt]: Generated new node with split x_131 > 0.05004814777826065 in train samples separation (46, 451)
[anndt]: Generating 4 new samples.
[anndt]: Generated new node with split x_301 > 0.06701800203651462 in train samples separation (5, 41)
[anndt]: Generating 44 new samples.
[anndt]: Generated new node with split x_288 > 0.03739006353521823 in train samples separation (2, 3)
[anndt]: Generating 25 new samples.
[anndt]: Statistics test passed at confidence level 0.05
[anndt]: Generating 25 new samples.
[anndt]: Statistics test passed at confidence level 0.05
[anndt]: Generating 6[...]<jupyter_text>Decision Tree Visualization (.pdf) highlighting extracted rules will be saved in the current directory Saving Vocabulary to interpret Decision Tree Visualization<jupyter_code>v = list(vectorizer.vocabulary_.values())
w = list(vectorizer.vocabulary_.keys())
voc = {}
for i, j in enumerate(v):
    voc[j] = w[i]
import csv
with open('voc Dec8.csv', 'w') as f:
    for key in voc.keys():
        f.write("%s,%s\n"%(key,voc[key]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12744560248674347, 'fraction_numerical': 0.023770341927226185, 'mean_word_length': 3.4868744872846595, 'pattern_counts': {'":': 18, '<': 24, '<?xml version=': 0, '>': 31, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '24854', 'license': 'Apache-2.0', 'n_tokens_mistral': 3953, 'n_tokens_neox': 3671, 'n_words': 804, 'path': 'Sentiment_Analysis_Ruleex_ANN-DT.ipynb', 'repo_name': 'rohancode/ruleex_modified', 'size': '10938'}