<jupyter_start><jupyter_text>Let's go through the intro machine learning tutorial:http://scikit-learn.org/stable/tutorial/basic/tutorial.htmlWe should have already installed sklearn if not then use:pip install -U scikit-learnOr follow the instructions at: http://scikit-learn.org/stable/install.html<jupyter_code>from sklearn import datasets
#install test sets that come wiht sklearn
iris = datasets.load_iris()
digits = datasets.load_digits()
print(digits.data)
digits.target
digits.images[0]
#Predict which digit is being represented by an image
from sklearn import svm
clf = svm.SVC(gamma=0.001, C=100.)
#gamma was set manually, this could be found automaticall with good values other tools
clf.fit(digits.data[:-1], digits.target[:-1]) 
from sklearn import datasets
import matplotlib.pyplot as plt
%matplotlib inline
#Load the digits dataset
digits = datasets.load_digits()

#Display the first digit
plt.figure(1, figsize=(3, 3))
plt.imshow(digits.images[-1], cmap=plt.cm.gray_r, interpolation='nearest')
plt.show()

# predict which digit this is
clf.predict(digits.data[-1:])<jupyter_output><empty_output><jupyter_text>Do you agree that this is an image of an 8?  Did something go wrong?  What could be done better. Model Persistence using pickle<jupyter_code>from sklearn import svm
from sklearn import datasets
clf = svm.SVC()
iris = datasets.load_iris()
X, y = iris.data, iris.target
clf.fit(X, y)  


import pickle
s = pickle.dumps(clf)
clf2 = pickle.loads(s)
clf2.predict(X[0:1])

from sklearn.externals import joblib
joblib.dump(clf, 'filename.pkl') 
#gives back the list of files
clf = joblib.load('filename.pkl')<jupyter_output><empty_output><jupyter_text>Conventions<jupyter_code>import numpy as np
from sklearn import random_projection




# default type is float64 unless specified
rng = np.random.RandomState(0)
X = rng.rand(10, 2000)
X = np.array(X, dtype='float32')

transformer = random_projection.GaussianRandomProjection()
X_new = transformer.fit_transform(X)

print ( "Data types for X = " + str(X.dtype) + " and X_new = "+ str(X_new.dtype))
from sklearn import datasets
from sklearn.svm import SVC
iris = datasets.load_iris()
clf = SVC()
clf.fit(iris.data, iris.target)  
list(clf.predict(iris.data[:3]))
clf.fit(iris.data, iris.target_names[iris.target]) 
list(clf.predict(iris.data[:3]))  
# refitting and updating parameters
import numpy as np
from sklearn.svm import SVC

rng = np.random.RandomState(0)
X = rng.rand(100, 10)
y = rng.binomial(1, 0.5, 100)
X_test = rng.rand(5, 10)

clf = SVC()
clf.set_params(kernel='linear').fit(X, y)  
print ("Predict x_test: "+str(clf.predict(X_test)))
clf.set_params(kernel='rbf').fit(X, y)  
print ("Predict x_test after refitting: "+str(clf.predict(X_test)))<jupyter_output>Predict x_test: [1 0 1 1 0]
Predict x_test after refitting: [0 0 0 1 0]
{'fraction_non_alphanumeric': 0.12197724039829302, 'fraction_numerical': 0.020625889046941678, 'mean_word_length': 5.587822014051523, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '231674', 'license': 'MIT', 'n_tokens_mistral': 1078, 'n_tokens_neox': 1030, 'n_words': 275, 'path': 'Lectures/Lecture 11 - Machine Learning/Lecture 11 - Machine learning - Intro Tutorial.ipynb', 'repo_name': 'soumya8888/big-data-python-class', 'size': '2812'}