<jupyter_start><jupyter_text>Exercise 1.1<jupyter_code>import numpy as np
from scipy import stats
import matplotlib.pyplot as plt
from jupyterthemes import jtplot
jtplot.style(theme="oceans16")
mu1, sigma1 = 0., np.sqrt(1./2)
mu2, sigma2 = 1, np.sqrt(1./2)
x = np.linspace(-3, 3, 3000)

p_x1 = stats.norm(mu1, sigma1).pdf(x)
p_x2 = stats.norm(mu2, sigma2).pdf(x)
plt.plot(x, p_x1)
plt.plot(x, p_x2)
plt.vlines(x=0.5, ymin=0, ymax=0.6, colors="red")
plt.title("1.1.Gaussian distribution")
plt.show()
x = np.linspace(-1,4,3000)
p_x1 = stats.uniform(0).pdf(x)
p_x2 = stats.uniform(2).pdf(x)
plt.plot(x, p_x1)
plt.plot(x, p_x2)
plt.vlines(x=1.5, ymin=0, ymax=1.1, colors="red")
plt.title("1.1.Uniform distribution")
plt.show()
x = np.linspace(-1, 4, 3000)
p_x1 = stats.uniform(0).pdf(x)
p_x2 = stats.uniform(0.5).pdf(x)
plt.plot(x, p_x1)
plt.plot(x, p_x2)
plt.vlines(x=0.75, ymin=0, ymax=1.1, colors="red")
plt.title("1.1.Uniform distribution 2")
plt.show()
x = np.linspace(-1,4,3000)
p_x1 = stats.uniform(0).pdf(x)
p_x2 = stats.uniform(0.5,2).pdf(x)
plt.plot(x, p_x1)
plt.plot(x, p_x2)
plt.vlines(x=1, ymin=0, ymax=1.1, colors="red")
plt.title("1.1.Uniform distribution 3")
plt.show()<jupyter_output><empty_output><jupyter_text>Exercise 1.2<jupyter_code>mu1, sigma1 = 0., np.sqrt(1./2)
mu2, sigma2 = 1, np.sqrt(1./2)
x = np.linspace(-3, 3, 3000)

p_x1 = stats.norm(mu1, sigma1).pdf(x)
p_x2 = stats.norm(mu2, sigma2).pdf(x)
x0 = (1-np.log(2))/2
print(stats.norm(mu1, sigma1).pdf(x0))
print(stats.norm(mu2, sigma2).pdf(x0))
plt.plot(x, p_x1)
plt.plot(x, p_x2)
plt.vlines(x=x0, ymin=0, ymax=0.6, colors="red")
plt.title("1.2.Decision boundary with loss matirx")
plt.show()
x = np.linspace(-1, 4, 3000)
p_x1 = stats.uniform(0).pdf(x)
p_x2 = stats.uniform(0.5, 2).pdf(x)
plt.plot(x, p_x1)
plt.plot(x, p_x2)
plt.vlines(x=0.75, ymin=0, ymax=1.1, colors="red")
plt.title("1.2 Decision boundary based on loss matrix 2")
plt.show()<jupyter_output><empty_output><jupyter_text>Prtools for Pythons  Exercise 1.6<jupyter_code>import numpy as np
import prtools as pr 
import matplotlib.pyplot as plt
from jupyterthemes import jtplot
jtplot.style(theme="onedork")
np.random.seed(0)
x = np.random.random([10,3])
for i in x:
    print(i)
print("np.mean(x):", np.mean(x))
print("np.mean(x, axis=0):", np.mean(x, axis=0))
print("np.mean(x, axis=1):", np.mean(x, axis=1))

print("np.std(x):", np.std(x))
print("np.std(x, axis=0):",np.std(x,axis=0))
print("np.std(x, axis=1):",np.std(x,axis=1))<jupyter_output>np.mean(x): 0.5811203282945502
np.mean(x, axis=0): [0.52101579 0.62614181 0.59620338]
np.mean(x, axis=1): [0.62225542 0.53814403 0.76434099 0.56802049 0.52155909 0.31332251
 0.87559575 0.68038903 0.30051624 0.62705973]
np.std(x): 0.2801740004228059
np.std(x, axis=0): [0.2651162  0.26483543 0.2986654 ]
np.std(x, axis=1): [0.06930698 0.09085388 0.23290636 0.16896147 0.35041796 0.3682133
 0.08193329 0.15497924 0.24021371 0.22880695]<jupyter_text>$\boldsymbol{NOTE:}$ \mean(x): the mean of all data \mean(x, axis=0): the mean of each column 3 \mean(x, axis=1): the mean of each row 10   Exercise 1.7<jupyter_code>plt.scatter(x[:,0],x[:,1])
plt.show()<jupyter_output><empty_output><jupyter_text>Exercise 1.8<jupyter_code>lab = np.array([1,1,1,1,2,2,2,2,2,2]).T
a = pr.prdataset(x,lab)
print(a)<jupyter_output>10 by 3 prdataset with 2 classes: [4 6]<jupyter_text>Exercise 1.9<jupyter_code>a = pr.boomerangs()
pr.scatterd(a[:,[1,2]])
print(a.shape)
a = pr.gendatb()
pr.scatterd(a)
print(a.shape)
b = +a #extract the original data matrix from a prdataset
b = pr.prdataset(+a)
pr.scatterd(b)
# train a nearest mean classiﬁer on the Banana dataset

a = pr.gendatb() # generate banana set
print("a.shape:", a.shape)
w = pr.nmc()
w.train(a)
print(w)
# When a mapping is trained, 
# it can be applied to a dataset using the operator *:
b = w.eval(a) #the classiﬁed, rescaled or mapped result of applying the mapping deﬁnition stored in w to a
print(b)
# labeld and testc are the general classiﬁcation and testing routines
lab = b*pr.labeld()
print(lab)
e = pr.testc(b)
print(e)<jupyter_output>0.2<jupyter_text>Available classifiers ldc Linear discriminant analysisqdc Quadratic discriminant analysisnmc Nearest mean classifierfisherc Fishers linear discriminantknnc k-nearest neighbor classifierparzenc Parzen classifiernaivebc Naive-Bayes classifiermogc Mixture-of-Gaussians classifierstumpc decision stump classifierdectreec Decision tree classifieradaboostc AdaBoostsvc Support vector classifier  Common classes gendatb Generation of banana shaped classesgendatc Generation of circular classesgendatd Generation of two difficult classesgendats Generation of two Gaussian distributed classesgendath Generation of the Higleyman dataset<jupyter_code>a = pr.gendath()
w = pr.parzenc(a)
pr.scatterd(a)
pr.plotc(w)
# SVC
w = pr.svc(a,('rbf',4.5,1))
print(w)
# specify an untrained mapping beforehand with the required hyperparameters
u = pr.svc(('rbf',4.5,1))
w =a*u
print(w)
# Or the other representation:
w = pr.svc(('rbf',4.5,1))
w.train(a)
print(w)<jupyter_output>Support vector classifier, 2 to 2 trained mapping<jupyter_text>Exercise 1.10 Create some of the artiﬁcial datasets, make scatterplots of the data, train some classiﬁers and plot their decision boundaries.<jupyter_code># Create some of the artiﬁcial datasets
Data = pr.gendats((50,50),dim=2,delta=3)
print("Data:", Data)
Classifier = pr.parzenc()
# train some classiﬁers
Classifier.train(Data)
Predict = Classifier.eval(Data)
print("Predict:", Predict)
e = pr.testc(Predict)
print(e)
# make scatterplots of the data
pr.scatterd(Data)

#plot their decision boundaries
pr.plotc(Classifier)
lab = +Predict[:,0]<=+Predict[:,1]
lab = np.where(lab == False, -1, 1)
assert((pr.labeld(Predict) == lab).all())
a = pr.prdataset(Predict,lab)
print(a)<jupyter_output>Simple dataset 100 by 2 prdataset with 2 classes: [50 50]<jupyter_text>Exercise 1.12<jupyter_code>from scipy.stats import multivariate_normal
def gauss_fun(X, Y):
    mux = [0]
    muy = [0]
    sx = [1]
    sy = [3]
    rho = [0]
    d = np.dstack([X,Y])
    z = None
    for i in range(len(mux)):
        mean = [mux[i],muy[i]]
        # Extract covariance matrix
        cov = [[sx[i] * sx[i], rho[i] * sx[i] *sy[i]], [rho[i] * sx[i] * sy[i], sy[i] * sy[i]]]
        gaussian = multivariate_normal(mean=mean,cov=cov)
        z_ret = gaussian.pdf(d)
        if z is None:
            z = z_ret
        else:
            z +=z_ret
    return z

x = np.linspace(-7, 7, 100)
y = np.linspace(-7, 7, 100)
X, Y = np.meshgrid(x, y)
Z = gauss_fun(X, Y)

plt.contour(X, Y, Z, colors='black');<jupyter_output><empty_output><jupyter_text>a) Ratio is 3. b) Covariance increases. Largest variance decreases while smallest goes up.<jupyter_code>def gauss_fun(X, Y):
    mux = [0]
    muy = [0]
    sx = [3] #\sqrt(ab)
    sy = [3]
    rho = [0.9]
    d = np.dstack([X,Y])
    z = None
    for i in range(len(mux)):
        mean = [mux[i],muy[i]]
        # Extract covariance matrix
        cov = [[sx[i] * sx[i], rho[i] * sx[i] *sy[i]], [rho[i] * sx[i] * sy[i], sy[i] * sy[i]]]
        gaussian = multivariate_normal(mean=mean,cov=cov)
        z_ret = gaussian.pdf(d)
        if z is None:
            z = z_ret
        else:
            z +=z_ret
    return z

x = np.linspace(-7, 7, 100)
y = np.linspace(-7, 7, 100)
X, Y = np.meshgrid(x, y)
Z = gauss_fun(X, Y)

plt.contour(X, Y, Z, colors='black');<jupyter_output><empty_output><jupyter_text>Exercise 1.13<jupyter_code>mySample = pr.numpy.random.randn(1000,2)
a = pr.prdataset(mySample) #Turn these random points into a prdataset a
w = pr.gaussm(a) #determines estimates for a normal distribution based on the data set a
pr.scatterd(a) #scatterd the data
pr.plotm(w) #visualize the 2D density estimates on top of the scattered data.<jupyter_output><empty_output><jupyter_text>Exercise 1.15 a) The optimal decision boundary has linear shape.<jupyter_code>myData = pr.gendats((10,10), dim=2, delta=1)
labelData = pr.labeld(myData)
# estimate linear discriminants by means of ldc(a) 
# and quadratic discriminant boundaries by means of qdc(a).
myLDC = pr.ldc()
myQDC = pr.qdc()
myLDC.train(myData)
myQDC.train(myData)

pr.scatterd(myData)
pr.plotc(myLDC)
pr.plotc(myQDC)<jupyter_output><empty_output><jupyter_text>b) \The boundary shape will be quadratic because of the limited training points. c) \LDC is by construction linear, while QDC is nonlinear as the decision boundary. d) \When the number of points per class increases, the solutions by ldc and qdc will coincide.\When there are two samples for each class, the number of the samples is not enough for QDC estimator.\When there is only one sample for each class, it is not enough for LDC estimator.\Therefore, the quantity of estimating samples for QDC and LDC is more than 2 and more than 1 respectively.  Exercise 1.16<jupyter_code>set1 = pr.numpy.random.randn(10,2)
label1 = np.zeros((1,10))
set2 = pr.numpy.random.randn(10,2)
label2 = np.ones((1,10))

setAll = np.vstack([set1,set2])
labelAll = np.vstack([label1.T,label2.T])

myData = pr.prdataset(setAll, labelAll)

myLDC = pr.ldc()
myQDC = pr.qdc()

myLDC.train(myData)
myQDC.train(myData)

pr.scatterd(myData)
pr.plotc(myLDC)
pr.plotc(myQDC)

errorLDC = pr.testc(myLDC(myData))
errorQDC = pr.testc(myQDC(myData))

# points on the wrong side of the boundary
print("errorLDC:", errorLDC*20)
print("errorQDC:", errorQDC*20)
a = myData
wLDC = myLDC
wQDC = myQDC
labelLDC = a*wLDC*pr.labeld()
labelQDC = a*wQDC*pr.labeld()

# Check your count from the previous question
true_errorLDC = 20 - np.sum(labelLDC == labelAll)
true_errorQDC = 20 - np.sum(labelQDC == labelAll)

print("true_errorLDC:",true_errorLDC)
print("true_errorQDC:",true_errorQDC)<jupyter_output>true_errorLDC: 7
true_errorQDC: 8<jupyter_text>Exercise 1.17<jupyter_code>a = pr.gendats((20,20),1,8)
#deﬁne the width parameter h for the Parzen kernel
h = 0.8 #optimal
#The function parzenm estimates a density for a given dataset.
#To deﬁne a Parzen density estimator with a certain width parameter h on the entire dataset,
#ignoring labels, type:
a = pr.prdataset(+a)
w = pr.parzenm(a,h)

pr.scatterd(a)
pr.plotm(w,gridsize=100) #If your graphs look a little “bumpy”, you can increase the grid size
help(pr.gendats)<jupyter_output>Help on function gendats in module prtools.prtools:

gendats(n, dim=2, delta=2.0)
    Generation of a simple classification data.
    
        A = gendats(N,DIM,DELTA)
    
    Generate a two-class dataset A from two DIM-dimensional Gaussian
    distributions, containing N samples. Optionally, the mean of the
    first class can be shifted by an amount of DELTA.<jupyter_text>Exercise 1.18<jupyter_code>a = pr.gendats([20,20],1,8) # Generate data
a = pr.prdataset(+a)
h =  [0.01,0.05,0.1,0.25,0.5,1,1.5,2,3,4,5]# Array of h's to try
LL = np.zeros(len(h))
for i in range(len(h)):
    w = pr.parzenm(a, h[i])
    LL[i] = np.sum(np.log(+(a*w)))
    # the estimated density pˆ for objects in a dataset a is given by +(a*w)

plt.plot(h,LL)
plt.ylabel("Log-likelihood")
plt.xlabel("h")
plt.show()

pr.scatterd(a)
pr.plotm(pr.parzenm(a,h[np.argmax(LL)]))<jupyter_output><empty_output><jupyter_text>Exercise 1.19<jupyter_code>import warnings
warnings.filterwarnings("ignore")

# Split into trn and tst, both 50%
[trn, tst] = pr.gendat(a, 0.5)

# Array of h’s to try
h = [0.01,0.05,0.1,0.25,0.5,1,1.5,2,3,4,5] 

Ltrn = np.zeros(len(h))
Ltst = np.zeros(len(h))

for i in range(len(h)):
    w = pr.parzenm(trn, h[i])
    Ltrn[i] = np.sum(np.log(+(trn*w)))
    Ltst[i] = np.sum(np.log(+(tst*w)))

plt.plot(h, Ltrn, "bo-")
plt.plot(h, Ltst, 'go-')
plt.ylabel("Log-likelihood")
plt.xlabel("h")
plt.legend("train", "test")
plt.show()
pr.scatterd(a)
pr.plotm(pr.parzenm(trn,h[4]),gridsize=100)<jupyter_output><empty_output><jupyter_text>Exercise 1.20<jupyter_code>help(pr.gendatb)
# Generate a simple 2D dataset (for instance, using gendatb) and 
# plot the decision boundaries of the six classiﬁers listed above. 
# Use k = 1 for the knnc.
dat = pr.prdataset(pr.gendatb(n=(50,50),s=2))
NMC = pr.nmc(dat)
LDC = pr.ldc(dat)
QDC = pr.qdc(dat)
FisherC = pr.fisherc(dat)
ParzenC = pr.parzenc(dat)
KNN = pr.knnc(dat,(1))

pr.scatterd(dat)
pr.plotc(NMC)
pr.plotc(LDC)
pr.plotc(QDC)
pr.scatterd(dat)
pr.plotc(FisherC)
pr.plotc(ParzenC)
pr.plotc(KNN)

for C in [NMC,LDC,QDC,FisherC,ParzenC,KNN]:
    print(str(C),":",pr.testc(C(dat)))
    print()
# Make a new dataset in which the second feature is 10 times larger than the dataset above. 
newtrain = dat;
newtrain[:,1] = 10*(+newtrain[:,1]).reshape(100)

NMC = pr.nmc(newtrain)
LDC = pr.ldc(newtrain)
QDC = pr.qdc(newtrain)
FisherC = pr.fisherc(newtrain)
ParzenC = pr.parzenc(newtrain)
KNN = pr.knnc(dat,(1))
pr.scatterd(newtrain)
pr.plotc(NMC)
pr.plotc(LDC)
pr.plotc(QDC)
pr.scatterd(newtrain)
pr.plotc(FisherC)
pr.plotc(ParzenC)
pr.plotc(KNN)
for C in [NMC,LDC,QDC,FisherC,ParzenC,KNN]:
    print(str(C),":",pr.testc(C(newtrain)))
    print()<jupyter_output>Nearest mean, 2 to 2 trained mapping : 0.56

LDA, 2 to 2 trained mapping : 0.22

QDA, 2 to 2 trained mapping : 0.2

Fisher, 2 to 2 trained mapping : 0.22

Parzen density+Bayes rule, 2 to 2 trained mapping : 0.0

1-NN+Bayes rule, 2 to 2 trained mapping : 0.0
{'fraction_non_alphanumeric': 0.15538029877910062, 'fraction_numerical': 0.05497838780617274, 'mean_word_length': 4.799472295514512, 'pattern_counts': {'":': 2, '<': 62, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '438687', 'license': 'MIT', 'n_tokens_mistral': 6003, 'n_tokens_neox': 5551, 'n_words': 1303, 'path': 'assignment/Assignment 1 Week 1.ipynb', 'repo_name': 'wyw970624/CS4220-machine-learning-1', 'size': '13187'}