<jupyter_start><jupyter_text>1- Vectors, Matrices and Arrays  sparse matrix<jupyter_code>import numpy as np
from scipy import sparse

mat = np.array([[0, 0],
              [0, 1],
              [3, 0]])
mat_sparse = sparse.csr_matrix(mat)
print(mat_sparse)

large_mat = np.array([
    [0,0,0,0,0,0,0,0,0,0,0,0],
    [0,1,0,0,0,0,0,0,0,0,0,0],
    [3,0,0,0,0,0,0,0,0,0,0,0]])
large_mat_sparse = sparse.csr_matrix(large_mat)
print(large_mat_sparse)<jupyter_output>(1, 1)	1
  (2, 0)	3
  (1, 1)	1
  (2, 0)	3<jupyter_text>apply function to mat elements<jupyter_code>import numpy as np

# vectorize (loop = no perf optimization)
mat = np.array([[1,2,3], [4,5,6], [7,8,9]])
add_100 = lambda x : x + 100
vec_add_100 = np.vectorize(add_100)
print(vec_add_100(mat))

# broadcasting
print(mat + 100)<jupyter_output>[[101 102 103]
 [104 105 106]
 [107 108 109]]
[[101 102 103]
 [104 105 106]
 [107 108 109]]<jupyter_text>min, max, avg, var, std<jupyter_code>import numpy as np

mat = np.array([[1,2,3], [4,5,6], [7,8,9]])
np.max(mat, axis=1)
np.mean(mat)
np.var(mat)
np.std(mat)<jupyter_output><empty_output><jupyter_text>reshaping<jupyter_code>import numpy as np

mat = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
mat.shape
mat.reshape(2,6)
mat.reshape(3, -1) # -1 == as many as needed<jupyter_output><empty_output><jupyter_text>transform<jupyter_code>import numpy as np

mat = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
mat.T<jupyter_output><empty_output><jupyter_text>flatten<jupyter_code>import numpy as np

mat = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
mat.flatten() # compare to mat.reshape(1, -1) ;-)<jupyter_output><empty_output><jupyter_text>rank<jupyter_code>import numpy as np

mat = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
np.linalg.matrix_rank(mat)<jupyter_output><empty_output><jupyter_text>determinant<jupyter_code>import numpy as np

mat = np.array([[1,2,3], [4,5,6], [7,8,9]])
np.linalg.det(mat)
mat = np.array([[1,2,3], [2,4,6], [3,8,9]])
np.linalg.det(mat)<jupyter_output><empty_output><jupyter_text>diagonal<jupyter_code>mat = np.array([[1,2,3], [4,5,6], [7,8,9]])
mat.diagonal()<jupyter_output><empty_output><jupyter_text>trace = sum of main diagonla<jupyter_code>mat = np.array([[1,2,3], [4,5,6], [7,8,9]])
mat.trace() == sum(mat.diagonal())<jupyter_output><empty_output><jupyter_text>eigenvalues & eingenvectors$Av = \lambda v$<jupyter_code>mat = np.array([[1,2,3], [4,5,6], [7,8,9]])
eigenvalues, eigenvectors = np.linalg.eig(mat)
print(f'eigenvalues = {eigenvalues}\neigenvectors = {eigenvectors}')<jupyter_output>eigenvalues = [ 1.61168440e+01 -1.11684397e+00 -1.30367773e-15]
eigenvectors = [[-0.23197069 -0.78583024  0.40824829]
 [-0.52532209 -0.08675134 -0.81649658]
 [-0.8186735   0.61232756  0.40824829]]<jupyter_text>dot product $\sum_{i=1}^{n} a_i b_i$<jupyter_code>import numpy as np

v1 = np.array([1,2,3])
v2 = np.array([4,5,6])

np.dot(v1, v2)
# python 3.5+
v1 @ v2<jupyter_output><empty_output><jupyter_text>matrix multiplication<jupyter_code>import numpy as np

mat1 = np.array([[1,2],[4,3]])
mat2 = np.array([[4,5],[7,6]])

np.dot(mat1, mat2)
mat1 @ mat2<jupyter_output><empty_output><jupyter_text>inverse$AA^{-1} = 1$<jupyter_code>import numpy as np

mat1 = np.array([[1,2],[4,3]])
np.linalg.inv(mat1)
mat1 @ np.linalg.inv(mat1)<jupyter_output><empty_output><jupyter_text>random matrix<jupyter_code>import numpy as np

np.random.seed(0)
np.random.uniform(1.0,2.0,3)<jupyter_output><empty_output><jupyter_text>Loading Data  load sample dataset<jupyter_code>from sklearn import datasets

digits = datasets.load_digits()
features = digits.data
target = digits.target

print(features[0])
print(len(target))<jupyter_output>[ 0.  0.  5. 13.  9.  1.  0.  0.  0.  0. 13. 15. 10. 15.  5.  0.  0.  3.
 15.  2.  0. 11.  8.  0.  0.  4. 12.  0.  0.  8.  8.  0.  0.  5.  8.  0.
  0.  9.  8.  0.  0.  4. 11.  0.  1. 12.  7.  0.  0.  2. 14.  5. 10. 12.
  0.  0.  0.  0.  6. 13. 10.  0.  0.  0.]
1797<jupyter_text>create simulated dataset  linear regression<jupyter_code>from sklearn.datasets import make_regression

features, target, coefficient = make_regression(
    n_samples=100,
    n_features=3,
    n_informative=3,
    n_targets=1,
    noise=0.0,
    coef=True,
    random_state=1)

print(f'features:\n{features[:3]}')
print(f'target:\n{target[:3]}')<jupyter_output>features:
[[ 1.29322588 -0.61736206 -0.11044703]
 [-2.793085    0.36633201  1.93752881]
 [ 0.80186103 -0.18656977  0.0465673 ]]
target:
[-10.37865986  25.5124503   19.67705609]<jupyter_text>classification<jupyter_code>from sklearn.datasets import make_classification

features, target = make_classification(
    n_samples=100,
    n_features=3,
    n_informative=3,
    n_redundant=0,
    n_classes=2,
    weights=[0.25,0.75],
    random_state=1)

print(f'features:\n{features[:3]}')
print(f'target:\n{target[:3]}')<jupyter_output>features:
[[ 1.06354768 -1.42632219  1.02163151]
 [ 0.23156977  1.49535261  0.33251578]
 [ 0.15972951  0.83533515 -0.40869554]]
target:
[1 0 0]<jupyter_text>clustering<jupyter_code>from sklearn.datasets import make_blobs

features, target = make_blobs(
    n_samples=100,
    n_features=2,
    centers=3,
    cluster_std=.5,
    shuffle=True,
    random_state=1
)
print(f'features:\n{features[:3]}')
print(f'target:\n{target[:3]}')
import matplotlib.pyplot as plt

plt.scatter(features[:,0], features[:,1], c=target)<jupyter_output><empty_output><jupyter_text>querying SQL database<jupyter_code>import pandas as pd
from sqlalchemy import create_engine
from sqlalchemy import Table, Column, Integer, String, MetaData

meta = MetaData()

students = Table(
   'students', meta, 
   Column('id', Integer, primary_key = True), 
   Column('name', String), 
   Column('lastname', String), 
)
meta.create_all(engine)

ins = students.insert()

engine = create_engine('sqlite:///sample.db')

with engine.connect() as conn:
    for fname, lname in [('fname1', 'lname1'), ('fname2', 'lname3')]:
        conn.execute(ins.values(name=fname, lastname=lname))

df = pd.read_sql_query('select * from students', engine)
df.head(2)<jupyter_output><empty_output><jupyter_text>2- Data Wrangling  dataframe<jupyter_code>import pandas as pd
url = "https://tinyurl.com/titanic-csv"

df = pd.read_csv(url)
df.head(5)
df.describe()
df.iloc[0]
df.iloc[:4]
df.loc[0]<jupyter_output><empty_output><jupyter_text>conditional selection<jupyter_code>df[df['Sex'] == 'female'].head(5)<jupyter_output><empty_output><jupyter_text>dataframe manipulation (not in place)  replace values<jupyter_code>df['Sex'].replace(['female', 'male'], ['woman', 'man']).head(5)<jupyter_output><empty_output><jupyter_text>rename columns<jupyter_code>df.rename(columns={'PClass': 'Passenger Class', 'Sex': 'Gender'}).head(5)
# get all column names in dataframe

import collections
column_names = collections.defaultdict(str)
for name in df.columns:
    column_names[name]
column_names<jupyter_output><empty_output><jupyter_text>count<jupyter_code>df.count()<jupyter_output><empty_output><jupyter_text>find unique values<jupyter_code>df['Sex'].unique()
df['Sex'].value_counts()
df['PClass'].value_counts()
df['PClass'].nunique() # number of unique values<jupyter_output><empty_output><jupyter_text>find missing data (null)<jupyter_code>df[df['Age'].isnull()].head(5)
# Nan comes from numpy

import numpy as np
df['Sex'].replace('male', np.nan).head(5)<jupyter_output><empty_output><jupyter_text>delete columns<jupyter_code>df.drop(['Age', 'Sex'], axis=1).head(3)<jupyter_output><empty_output><jupyter_text>delete rows<jupyter_code># just filter!
df[df['Sex'] == 'male'].head(5)<jupyter_output><empty_output><jupyter_text>remove duplicate rows<jupyter_code>df.drop_duplicates(subset=['Sex'], keep='last').head(4)<jupyter_output><empty_output><jupyter_text>group rows by value<jupyter_code>df.groupby(['Survived', 'PClass']).count()<jupyter_output><empty_output><jupyter_text>group by time<jupyter_code>import pandas as pd
import numpy as np

time_index = pd.date_range('06.06.2018', periods=100000, freq='30S')
df_t = pd.DataFrame(index=time_index)
df_t['Sale_Amount'] = np.random.randint(1, 10, 100000)
print(df_t.head(5))
print(df_t.resample('W').count())
print(df_t.resample('W', label='left').count())<jupyter_output>Sale_Amount
2018-06-06 00:00:00            8
2018-06-06 00:00:30            9
2018-06-06 00:01:00            6
2018-06-06 00:01:30            2
2018-06-06 00:02:00            3
            Sale_Amount
2018-06-10        14400
2018-06-17        20160
2018-06-24        20160
2018-07-01        20160
2018-07-08        20160
2018-07-15         4960
            Sale_Amount
2018-06-03        14400
2018-06-10        20160
2018-06-17        20160
2018-06-24        20160
2018-07-01        20160
2018-07-08         4960<jupyter_text>lopping over column<jupyter_code>for name in df['Name'].head(3):
    print(name)
[name.upper() for name in df['Name'][:3]]<jupyter_output><empty_output><jupyter_text>apply lambda to column values<jupyter_code>df['Name'].apply(lambda x: x.upper())[:3]<jupyter_output><empty_output><jupyter_text>apply function to groups<jupyter_code>print(df.groupby('Sex').apply(lambda x: x.count()))
print(df.groupby('Sex').count())<jupyter_output>Name  PClass  Age  Sex  Survived  SexCode
Sex                                              
female   462     462  288  462       462      462
male     851     851  468  851       851      851
        Name  PClass  Age  Survived  SexCode
Sex                                         
female   462     462  288       462      462
male     851     851  468       851      851<jupyter_text>concat dataframes<jupyter_code>import pandas as pd

data_a = {
    'id' : ['1', '2', '3'],
    'first': ['Alex', 'Amy', 'Allen'],
    'last': ['Anderson', 'Ackman', 'Ali']
}
df_a = pd.DataFrame(data_a, columns=['id', 'first', 'last'])

data_b = {
    'id' : ['4', '5', '6'],
    'first': ['Billy', 'Brian', 'Bran'],
    'last': ['Bonder', 'Black', 'Balwner']
}
df_b = pd.DataFrame(data_b, columns=['id', 'first', 'last'])

pd.concat([df_a, df_b], axis=1)<jupyter_output><empty_output><jupyter_text>merge dataframes (joins)<jupyter_code>import pandas as pd

employee_data = {
    'employee_id': ['1', '2', '3', '4'],
    'name': ['<NAME>', '<NAME>', '<NAME>', '<NAME>']
}
df_employees = pd.DataFrame(employee_data, columns=['employee_id', 'name'])

sales_data = {
    'employee_id': ['3', '4', '5', '6'],
    'total_sales': [23456, 2512, 2345, 1455]
}
df_sales = pd.DataFrame(sales_data, columns=['employee_id', 'total_sales'])
# inner join
pd.merge(df_employees, df_sales, on='employee_id')
# outer join
pd.merge(df_employees, df_sales, on='employee_id', how='outer')
# custom join
pd.merge(df_employees, df_sales, left_on='employee_id', right_on='employee_id')<jupyter_output><empty_output><jupyter_text>4- handling numerical data  rescale a feature$x'_i = \frac{x_i - min(x)}{max(x) - min(x)}$<jupyter_code>import numpy as np
from sklearn import preprocessing

feat = np.array([
    [-500.5],
    [-100.1],
    [0],
    [100.1],
    [900.9],
])
minmax_scale = preprocessing.MinMaxScaler(feature_range=(0,1))
scaled_feat = minmax_scale.fit_transform(feat)
scaled_feat<jupyter_output><empty_output><jupyter_text>standardizing a feature$x'_i = \frac{x_i - \bar{x}}{\sigma}$<jupyter_code>import numpy as np
from sklearn import preprocessing

feat = np.array([
    [-1000.1],
    [-200.2],
    [500.5],
    [600.6],
    [9000.9],
])
scaler = preprocessing.StandardScaler()
std_feat = scaler.fit_transform(feat)
std_feat<jupyter_output><empty_output><jupyter_text>normalize observations Euclidean norm:$\|{x}\|_2 = \sqrt{x^2_1+x^2_2+x^2_3+...+x^2_n}$<jupyter_code>import numpy as np
from sklearn.preprocessing import Normalizer

feats = np.array([
    [0.5, 0.5],
    [1.1, 3.4],
    [1.5, 20.2],
    [1.63, 34.4],
    [10.9, 3.3],
])
Normalizer(norm="l2").transform(feats)<jupyter_output><empty_output><jupyter_text>Manhattan norm:$\|x\|_1 = \sum^n_{i=1}{\|x_i\|}$<jupyter_code>Normalizer(norm="l1").transform(feats)<jupyter_output><empty_output><jupyter_text>generate polynomial featues<jupyter_code>import numpy as np
from sklearn.preprocessing import PolynomialFeatures

feats = np.array([
    [2,3],
    [2,3],
    [2,3],
])
polynomial_interaction = PolynomialFeatures(degree=2, include_bias=False)

polynomial_interaction.fit_transform(feats)<jupyter_output><empty_output><jupyter_text>custom transform features<jupyter_code>import numpy as np
from sklearn.preprocessing import FunctionTransformer

feats = np.array([
    [2,3],
    [2,3],
    [2,3],
])
FunctionTransformer(lambda x : x + 10, validate=True).transform(feats)
# with pandas

import pandas as pd

df = pd.DataFrame(feats, columns=["feat1", "feat2"])
df.apply(lambda x:x+10)<jupyter_output><empty_output><jupyter_text>detecting outliners<jupyter_code>import numpy as np
from sklearn.covariance import EllipticEnvelope
from sklearn.datasets import make_blobs

feats, _ = make_blobs(
    n_samples=10,
    n_features=2,
    centers=1,
    random_state=1)
feats[0,0] = 10000
feats[0,1] = 10000
print(feats)
outliner_detector = EllipticEnvelope(contamination=.1)
outliner_detector.fit(feats)
outliner_detector.predict(feats)
# interquartal range IQR

feat = feats[:,0]

# find outliners index
def indicies_of_outliners(x):
    q1, q3 = np.percentile(x, [25, 75])
    iqr = q3 - q1
    lower_bound = q1 - (iqr * 1.5)
    upper_bound = q3 + (iqr * 1.5)
    return np.where((x > upper_bound) | (x < lower_bound))

indicies_of_outliners(feat)<jupyter_output><empty_output><jupyter_text>handling outliners  drop them<jupyter_code>import pandas as pd

houses = pd.DataFrame()
houses['Price'] = [534433, 392333, 293222, 43280232]
houses['Bathrooms'] = [2,3.5, 2, 116]
houses['Square_Feet'] = [1500, 2500, 1500, 48000]

# filter observations
houses[houses['Bathrooms'] < 20]
import numpy as np

houses['Outliner'] = np.where(houses['Bathrooms'] < 20, 0, 1)
houses
# dampen the effect of the outliner
houses['Log_of_Square_Feet'] = [np.log(x) for x in houses['Square_Feet']]
houses<jupyter_output><empty_output><jupyter_text>discretizing features<jupyter_code>import numpy as np
from sklearn.preprocessing import Binarizer

age = np.array([
    [6],
    [12],
    [20],
    [36],
    [65],
])
binarizer = Binarizer(18)
binarizer.fit_transform(age)
np.digitize(age, bins=[20, 30, 64])<jupyter_output><empty_output><jupyter_text>clustering features<jupyter_code># k-means clustering

import pandas as pd
from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans

feats, _ = make_blobs(
    n_samples=50,
    n_features=2,
    centers=3,
    random_state=1
)

df = pd.DataFrame(feats, columns=["feat1", "feat2"])
clusterer = KMeans(3, random_state=0)
clusterer.fit(feats)
df["groups"] = clusterer.predict(feats)
df.head(10)
df.plot()<jupyter_output><empty_output><jupyter_text>drop observations with missing values<jupyter_code># numpy

import numpy as np

feats = np.array([
    [1.1, 11.1],
    [2.2, 22.2],
    [3.3, 33.3],
    [4.4, 44.4],
    [np.nan, 55.5],
])
feats[~np.isnan(feats).any(axis=1)]
# pandas

import pandas as pd

df = pd.DataFrame(feats, columns=["feat1", "feat2"])
df.dropna()
# k-nearest neighbors (KNN)

import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_blobs

feats, _ = make_blobs(n_samples=1000,
                     n_features=2,
                     random_state=1)

# Standardize the features
scaler = StandardScaler()
standardized_features = scaler.fit_transform(feats)
# Replace the first feature's first value with a missing value
true_value = standardized_features[0,0]
standardized_features[0,0] = np.nan

mean_imputer = SimpleImputer(strategy="mean")
feats_mean_imputed = mean_imputer.fit_transform(feats)

print(f"actual value: {true_value}")
print(f"Imputed value: {feats_mean_imputed[0,0]}")<jupyter_output>actual value: 0.8730186113995938
Imputed value: -3.058372724614996<jupyter_text>5- Handling Categorical Data (Skipped)  6- Hanlding Text  cleaning text<jupyter_code>text_data = ["     Lorem Ipsum is simply dummy text     ",
             "of the printing and typesetting industry",
             "   Lorem Ipsum has been the industry's standard dummy    "]

[x.strip() for x in text_data]
[x.strip().upper() for x in text_data]
import re
[re.sub(r"[a-zA-Z]", "X", x).strip() for x in text_data]<jupyter_output><empty_output><jupyter_text>claening HTML<jupyter_code>from bs4 import BeautifulSoup

html = """
<div class='full_name'><span style='font-weight:bold'>Masego</span> Azra</div>
"""
soup = BeautifulSoup(html)
soup.find("div", {"class": "full_name"}).text<jupyter_output><empty_output><jupyter_text>remove punctuations<jupyter_code>import unicodedata
import sys

text_data = ['Hi!!!! I. Love. This. Song....',
             '10000% Agree!!!! #LoveIT',
             'Right?!?!']
punctuations = dict.fromkeys(i for i in range(sys.maxunicode)
                            if unicodedata.category(chr(i)).startswith('P'))
punctuations
[x.translate(punctuations) for x in text_data]<jupyter_output><empty_output><jupyter_text>tokenizing text<jupyter_code>from nltk.tokenize import word_tokenize

string = "The science of today is the technology of tomorrow"

word_tokenize(string)
from nltk.tokenize import sent_tokenize

string = "The science of today is the technology of tomorrow. Tomorrow is today"

sent_tokenize(string)<jupyter_output><empty_output><jupyter_text>----<jupyter_code>import numpy as np
import pandas as pd

data = [
    [8,13,3.5],
    [10,12,3.1],
    [9,11,2.8],
    [8,14,1.5],
    [14,14,1.2],
]
df = pd.DataFrame(data, columns=["Age", "Score 1", "Score 2"])
df.loc[df['Age']==9, ['Score 1', 'Score 2']]
import cv2<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16310537294920224, 'fraction_numerical': 0.08073518633365281, 'mean_word_length': 4.9126666666666665, 'pattern_counts': {'":': 1, '<': 215, '<?xml version=': 0, '>': 213, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '53', 'dir': 'jupyter-structured-clean-dedup', 'id': '376468', 'license': 'MIT', 'n_tokens_mistral': 7964, 'n_tokens_neox': 7166, 'n_words': 1307, 'path': 'ML.ipynb', 'repo_name': 'ammarnajjar/python-playground', 'size': '17752'}