<jupyter_start><jupyter_text>Problem Set 2<jupyter_code>NUM_CASES = 2000
#Setup
import warnings; warnings.simplefilter('ignore')
import matplotlib.pyplot as plt
%matplotlib inline
import pandas as pd
import os
import numpy as np

df1 = pd.read_csv('case_metadata.csv')[['caseid','case_reversed','judge_id','year','log_cites']]

#delete rows with NaN
df1 = df1[np.isfinite(df1['log_cites'])]


from random import shuffle
keep = [True] * NUM_CASES + [False] * (len(df1) - NUM_CASES)
shuffle(keep)
df1 = df1[keep]
print('Number of rows: ',len(df1))
df1.head()
# load text documents
tmp=[]
for i in range(len(df1)):
    caseid=df1.iloc[i][0]
    caseid=caseid+'.txt'
    txt_file = [f for f in os.listdir('cases/') if f.endswith(caseid)]
    path='cases/'+txt_file[0]
    txt = open(path, 'r').read() # open a document
    tmp.append(txt)
df1['text']=tmp
df1.head()
# Capitalization
def capitalization(doc):
    return doc.lower()
df1['doc'] = df1['text'].apply(capitalization) # go to lower-case
df1.head()
#####
# Punctuation
#####

# recipe for fast punctuation removal
from string import punctuation

def remove_punctuation(doc):
    translator = str.maketrans('','',punctuation) 
    return doc.translate(translator)

df1['doc'] = df1['doc'].apply(remove_punctuation)
df1.head()
# Tokens
def tokenize(doc):
    return doc.split()
df1['doc'] = df1['doc'].apply(tokenize)
df1.head()
# remove numbers (keep if not a digit)
def remove_numbers(doc):
    return [t for t in doc if not t.isdigit()]
df1['doc'] = df1['doc'].apply(remove_numbers)
df1.head()
# Stopwords
from nltk.corpus import stopwords
stoplist = stopwords.words('english') 
def remove_stopwords(doc):
    return [t for t in doc if t not in stoplist]
df1['doc'] = df1['doc'].apply(remove_stopwords)
df1.head()
# Stemming
from nltk.stem import SnowballStemmer
stemmer = SnowballStemmer('english') # snowball stemmer, english
def stemming(doc):
    return [stemmer.stem(t) for t in doc]
df1['doc'] = df1['doc'].apply(stemming)
df1.head()
# Lemmatizing
import nltk
from nltk.stem import WordNetLemmatizer
#nltk.download('wordnet')
wnl = WordNetLemmatizer()
def lemmatizing(doc):
    return [wnl.lemmatize(t) for t in doc]
#wnl.lemmatize('corporation'), wnl.lemmatize('corporations')
df1['doc'] = df1['doc'].apply(lemmatizing)
df1.head()
# remove tokens
def remove_tokens(doc):
    return " ".join(doc)
df1['doc'] = df1['doc'].apply(remove_tokens)
df1.head()
# Make a small data set
from sklearn.feature_extraction.text import CountVectorizer
vectorizer500 = CountVectorizer(stop_words='english',
                             max_features=500)
X500 = vectorizer500.fit_transform(df1['doc']).todense()
X500 = X500 / X500.sum(axis=1) # counts to frequencies
for i, word in enumerate(vectorizer500.get_feature_names()):
    column = X500[:,i]
    df1['x_'+word] = column


features = [x for x in df1.columns if x.startswith('x_')]


# set up train/test split
from sklearn.model_selection import train_test_split
train, test = train_test_split(df1, test_size=0.2)
df1.head()
from sklearn.linear_model import LogisticRegression
log_reg = LogisticRegression(solver='lbfgs', max_iter=3000)
log_reg.fit(train[features], train['case_reversed'])
log_reg.score(test[features],test['case_reversed'])<jupyter_output><empty_output><jupyter_text>1)Take your best-performing logistic regression model from Problem Set 1 (or train a new one). Use cross_val_predict() to form model predictions for each observation. Construct a confusion matrix for the predictions. Report the precision_recall_curve() and roc_curve(). Make a calibration plot (as in Bansak et al 2018, appendix page 24).<jupyter_code>from sklearn.model_selection import cross_val_predict
from sklearn.metrics import confusion_matrix,precision_recall_curve,roc_curve

# Form cross-validated prdictions for all data points
y_pred = cross_val_predict(log_reg, df1[features], df1['case_reversed'], cv=3)
# Construct confusion matrix
conf_matrix = confusion_matrix(df1['case_reversed'], y_pred)
probas_pred = log_reg.predict_proba(test[features])
probas_pred = probas_pred[:, 1]
precision, recall, thresholds = precision_recall_curve(test['case_reversed'], probas_pred)
import matplotlib.pyplot as plt
plt.plot(thresholds, precision[:-1], label="Precision")
plt.plot(thresholds, recall[:-1], label="Recall")
plt.xlabel('Threshold')
plt.legend()
# calculate roc curve
from sklearn.metrics import roc_curve
fpr, tpr, thresholds = roc_curve(test['case_reversed'], probas_pred)
# plot no skill
plt.plot([0, 1], [0, 1], linestyle='--')
# plot the roc curve for the model
plt.plot(fpr, tpr, marker='.')
# show the plot
plt.show()<jupyter_output><empty_output><jupyter_text>2)Scale your n-gram frequencies while maintaining sparsity, as discussed in class. Train an elastic net model to predict log citations to a case (log_cites). Run cross_val_predict() to form model predictions and report a scatter plot of true and predicted values.<jupyter_code># scikit-learn text vectorizer
from sklearn.feature_extraction.text import CountVectorizer
vec = CountVectorizer(min_df=0.01, # at min 1% of docs
                        max_df=.9,  
                        max_features=500,
                        stop_words='english',
                        ngram_range=(1,3))
Xraw = vec.fit_transform(df1['doc'])
# Scaling with Sparsity
from sklearn.preprocessing import StandardScaler
sparse_scaler = StandardScaler(with_mean=False)
X_sparse = sparse_scaler.fit_transform(Xraw)
###
# Elastic Net
###
from sklearn.linear_model import ElasticNetCV
enet_reg = ElasticNetCV(alphas=[.0001, .001, .01,.1,1], 
                        l1_ratio=[.0001, .001, .01,.1,.5,.9, .99, 1])
enet_reg.fit(train[features],train['log_cites'])
enet_reg.alpha_, enet_reg.l1_ratio_
y_pred = cross_val_predict(enet_reg,df1[features], df1['log_cites'], cv=3)
plt.scatter(y_pred,df1['log_cites'])#alpha=.1
plt.show()<jupyter_output><empty_output><jupyter_text>3)Use the judge identifiers (judge_id) to produce the average citations to cases for each judge. Then estimate a two-stage least-squares model with Z as average citations of the judge, X as citations to the case, and Y as whether the case was reversed. Include year fixed effects. Report estimates for coefficient and standard error on X.<jupyter_code>#calculate average citations to cases for each judge
df2 = df1.drop_duplicates(subset='judge_id', keep='first').loc[:,['judge_id']]
df2['average_citations']=[df1.loc[df1['judge_id'] == i]['log_cites'].mean() for i in df2['judge_id']]
df2.head()<jupyter_output><empty_output><jupyter_text>4)Train LDA on the cases. Produce word clouds for the topics. Inspect the word clouds for different models to decide on the right number of topics.<jupyter_code>text=""
for doc in df1.iloc[:,6]:
    text+=doc
#from sklearn.lda import LDA
from wordcloud import WordCloud

wordcloud = WordCloud().generate(text)
# Display the generated image:
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>5)Replicate your sklearn logistic regression model (no hidden layers, L2 regularization, and softmax output layer) in Keras or some other deep learning library. Compare performance. Now add two hidden layers, dropout, and batch normalization. Compare performance.<jupyter_code>from keras.models import Sequential
from keras.layers import Dense
from keras import regularizers

X=Xraw.toarray()
Y=df1['case_reversed']

model = Sequential() # create a sequential model
model.add(Dense(10, # output neurons in layer       
          input_dim=X.shape[1], # number of inputs
          kernel_regularizer=regularizers.l2(0.01),
          activity_regularizer=regularizers.l2(0.01),
          activation='relu')) # activation function
model.add(Dense(1, activation='softmax')) # output layer
model.summary()
# Visualize a model

# Requires graphviz
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
dot = model_to_dot(model,
                   show_shapes=True,
                   show_layer_names=False)
SVG(dot.create(prog='dot', format='svg'))
# fit the model
model.compile(loss='binary_crossentropy', # cost function
              optimizer='adam', # use adam as the optimizer
              metrics=['accuracy']) # compute accuracy, for scoring

model_info = model.fit(X, Y, 
                      epochs=10,
                      validation_split=.2)
# Plot performance by epoch
plt.plot(model_info.epoch,model_info.history['loss'])
plt.plot(model_info.epoch,model_info.history['val_loss'])
plt.legend(['loss', 'val_loss'], loc='best')
from keras.layers import Dropout
from keras.layers.normalization import BatchNormalization


model2 = Sequential() # create a sequential model
model2.add(Dense(5, # output neurons in layer       
          input_dim=X.shape[1], # number of inputs
          activation='relu')) # activation function
model2.add(Dense(5, activation='relu')) # hidden layer
model2.add(BatchNormalization())
model2.add(Dense(5, activation='relu')) # hidden layer
model2.add(Dropout(0.5))                  # dropout
model2.add(Dense(1, activation='softmax')) # output layer
model2.summary()
# Early stopping
from keras.callbacks import EarlyStopping
earlystop = EarlyStopping(monitor='val_acc', 
                          min_delta=0.0001, 
                          patience=5, 
                          mode='auto')

model2.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])

model2.fit(X, Y, batch_size=32, 
           epochs=100, 
           callbacks=[earlystop], 
           validation_split=0.2)<jupyter_output>Train on 1600 samples, validate on 400 samples
Epoch 1/100
1600/1600 [==============================] - 2s 1ms/step - loss: 7.8417 - acc: 0.5081 - val_loss: 1.1921e-07 - val_acc: 1.0000
Epoch 2/100
1600/1600 [==============================] - 0s 235us/step - loss: 7.8417 - acc: 0.5081 - val_loss: 1.1921e-07 - val_acc: 1.0000
Epoch 3/100
1600/1600 [==============================] - 0s 268us/step - loss: 7.8417 - acc: 0.5081 - val_loss: 1.1921e-07 - val_acc: 1.0000
Epoch 4/100
1600/1600 [==============================] - 0s 242us/step - loss: 7.8417 - acc: 0.5081 - val_loss: 1.1921e-07 - val_acc: 1.0000
Epoch 5/100
1600/1600 [==============================] - 0s 241us/step - loss: 7.8417 - acc: 0.5081 - val_loss: 1.1921e-07 - val_acc: 1.0000
Epoch 6/100
1600/1600 [==============================] - 0s 224us/step - loss: 7.8417 - acc: 0.5081 - val_loss: 1.1921e-07 - val_acc: 1.0000
{'fraction_non_alphanumeric': 0.14014071116181784, 'fraction_numerical': 0.04173797299866895, 'mean_word_length': 4.589266737513284, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '56210', 'license': 'MIT', 'n_tokens_mistral': 3822, 'n_tokens_neox': 3450, 'n_words': 886, 'path': 'ProblemSet2.ipynb', 'repo_name': 'ImreKertesz/BuildingRobotJudge', 'size': '10518'}