<jupyter_start><jupyter_text>Voting Vs. Stacking ensembles<jupyter_code>import numpy as np

from sklearn.datasets import fetch_mldata
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
mnist = fetch_mldata( "MNIST original" )
XTV, testX, yTV, testY = train_test_split( mnist.data, mnist.target, test_size = 10000,
                                           random_state = 123 )
trainX, valX, trainY, valY = train_test_split( XTV, yTV, test_size = 10000,
                                              random_state = 123 )<jupyter_output><empty_output><jupyter_text>Voting ensemble<jupyter_code>forest = RandomForestClassifier()
forest.fit( trainX, trainY )

print("Trained random forest.")
knn = KNeighborsClassifier()
knn.fit( trainX, trainY )

print("Trained KNN.")
logReg = LogisticRegression()
logReg.fit( trainX, trainY )

print("Trained logistic regression.")
#gaussian = GaussianNB()
#gaussian.fit( trainX, trainY ) This seems bad, see below

#print("Trained naive Bayes classifier")
mlp = MLPClassifier()
mlp.fit( trainX, trainY )

print("Trained MLP.")
pred1 = forest.predict( valX )
acc1 = accuracy_score( valY, pred1 )
pred2 = knn.predict( valX )
acc2 = accuracy_score( valY, pred2 )
pred3 = logReg.predict( valX )
acc3 = accuracy_score( valY, pred3 )
#pred3 = gaussian.predict( valX )
#acc3 = accuracy_score( valY, pred3 ) only 55%
pred4 = mlp.predict( valX )
acc4 = accuracy_score( valY, pred4 )

print( f"Random forest: {acc1}" + "\n"
       f"KNN: {acc2}" + "\n"
       f"Logistic regression: {acc3}" + "\n"
       f"MLP: {acc4}" )
hard = VotingClassifier( estimators = [ ("forest", forest),
                                        ("knn", knn),
                                        ("logreg", logReg),
                                        ("mlp", mlp) ],
                         voting = "hard" )
hard.fit( trainX, trainY )

print("Trained hard voter")
soft = VotingClassifier( estimators = [ ("forest", forest),
                                        ("knn", knn),
                                        ("logreg", logReg),
                                        ("mlp", mlp) ],
                         voting = "soft" )
soft.fit( trainX, trainY )

print("Trained soft voter")
predHard = hard.predict( valX )
accHard  = accuracy_score( valY, predHard )
predSoft = soft.predict( valX )
accSoft  = accuracy_score( valY, predSoft )

print( f"Hard vote: {accHard}\nSoft Vote: {accSoft}" )<jupyter_output>Hard vote: 0.9667
Soft Vote: 0.9698<jupyter_text>Stacking ensemble<jupyter_code>l2Data = np.array( [pred1, pred2, pred3, pred4] )
l2Data = l2Data.T
forest2 = RandomForestClassifier()
forest2.fit( l2Data, valY )

mlp2 = MLPClassifier()
mlp2.fit( l2Data, valY )
l2Test = np.array( [ forest.predict( testX ), knn.predict( testX ),
                     logReg.predict( testX ), mlp.predict( testX ) ] )
l2Test = l2Test.T
predStack = forest2.predict( l2Test )
#predStack = mlp2.predict( l2Test ) #This one is slightly worse. no tunning was done...
accStack = accuracy_score( testY, predStack )
print(f"Voting: {accSoft}\nStacking: {accStack}")<jupyter_output>Voting: 0.9698
Stacking: 0.9677
{'fraction_non_alphanumeric': 0.11641615089890953, 'fraction_numerical': 0.02269378131447097, 'mean_word_length': 2.983568075117371, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '541626', 'license': 'MIT', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1107, 'n_words': 278, 'path': 'Chapter07/ex8and9.ipynb', 'repo_name': 'm0baxter/MLBookStuff', 'size': '3393'}