<jupyter_start><jupyter_text><jupyter_code>import pandas as pd

tokyo = 'https://gist.githubusercontent.com/gumdropsteve/40555dbb4cb01e694328879175e5feaf/raw/59a74ffbb4000a37f1bdb2f61dc7e702e07fa6e7/tokyo.csv'

df = pd.read_csv(tokyo)

df.tail()
df.ds.unique()
df.ds.str.split(expand=True)[0]
df.ds.str.split(expand=True)[0].unique()<jupyter_output><empty_output><jupyter_text>Categorical Features<jupyter_code>data = [
    {'price': 850000, 'rooms': 4, 'neighborhood': 'Queen Anne'},
    {'price': 700000, 'rooms': 3, 'neighborhood': 'Fremont'},
    {'price': 650000, 'rooms': 3, 'neighborhood': 'Wallingford'},
    {'price': 600000, 'rooms': 2, 'neighborhood': 'Fremont'}
]

pd.DataFrame(data)<jupyter_output><empty_output><jupyter_text>https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.DictVectorizer.html<jupyter_code>from sklearn.feature_extraction import DictVectorizer

vec = DictVectorizer(sparse=False, dtype=int)

vec.fit_transform(data)
len(vec.fit_transform(data))
type(vec.fit_transform(data))
vec.fit_transform(data)[0]
pd.DataFrame(vec.fit_transform(data))
vec.get_feature_names()
pd.DataFrame(data)
t = vec.fit_transform(data)
names = vec.get_feature_names()

pd.DataFrame(t, columns=names)<jupyter_output><empty_output><jupyter_text>What if same price?<jupyter_code>data = [
    {'price': 850000, 'rooms': 4, 'neighborhood': 'Queen Anne'},
    {'price': 700000, 'rooms': 3, 'neighborhood': 'Fremont'},
    {'price': 700000, 'rooms': 3, 'neighborhood': 'Wallingford'},
    {'price': 650000, 'rooms': 3, 'neighborhood': 'Wallingford'},
    {'price': 600000, 'rooms': 2, 'neighborhood': 'Fremont'}
]

pd.DataFrame(data)
t = vec.fit_transform(data)
names = vec.get_feature_names()

pd.DataFrame(t, columns=names)<jupyter_output><empty_output><jupyter_text>Now with actual data...<jupyter_code>df['type']
len(df['type'].unique())
df['type'].value_counts()
# overall, we want to predict price
df['price']
# df.to_dict()
# from sklearn.feature_extraction import DictVectorizer

# vec = DictVectorizer(sparse=False, dtype=int)

# vec.fit_transform(df['type'])  # df['type'] is not a dict
d = {'type':t for t in df['type']}
d
# need my data to look like this
data
d = [{'type':v} for v in df.type]
# d
pd.DataFrame(d)
from sklearn.feature_extraction import DictVectorizer

vec = DictVectorizer(sparse=False, dtype=int)

# get list of dicts from `type` column
data = [{'type':v} for v in df.type]

vec.fit_transform(data)[0]
vec.feature_names_
vec.fit_transform(data)[-5:]
l = [1,2,7,32,8,2]
pd.Series(l)

pd.DataFrame(df['type'])
len(df['type'].unique())
from sklearn.feature_extraction import DictVectorizer

vec = DictVectorizer(sparse=False, dtype=int)

# get list of dicts from `type` column
data = [{'type':v} for v in df.type]

t = vec.fit_transform(data)
names = vec.feature_names_

pd.DataFrame(t, columns=names)
from sklearn.feature_extraction import DictVectorizer

vec = DictVectorizer(sparse=False, dtype=int)

# get list of dicts from `type` column
data = [{'type':v} for v in df.location]

t = vec.fit_transform(data)
names = vec.feature_names_

pd.DataFrame(t, columns=names)<jupyter_output><empty_output><jupyter_text>Note on pandas get_dummies...

https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.get_dummies.html<jupyter_code>pd.get_dummies(df['type'])
pd.get_dummies(df['type']).head(), df['type'].head()
pd.get_dummies(df['type']).tail(), df['type'].tail()<jupyter_output><empty_output><jupyter_text>Text Features<jupyter_code>sample = ['problem of evil',
          'evil queen',
          'horizon problem']
sample
from sklearn.feature_extraction.text import CountVectorizer

vec = CountVectorizer()
X = vec.fit_transform(sample)
X
# note, CountVectorizer returns sparce matrix by default
pd.DataFrame(X)
# so convert it to an array
X.toarray()
pd.DataFrame(X.toarray(), columns=vec.get_feature_names())
sample = ['problem of evil',
          'problem of evil again again 2 2 22',
          'evil queen',
          'horizon problem']

vec = CountVectorizer()
X = vec.fit_transform(sample)

pd.DataFrame(X.toarray(), columns=vec.get_feature_names())<jupyter_output><empty_output><jupyter_text>Now with actual data...<jupyter_code>df.title
vec = CountVectorizer()
X = vec.fit_transform(df.title)

pd.DataFrame(X.toarray(), columns=vec.get_feature_names())
import numpy as np

np.sum(pd.DataFrame(X.toarray(), columns=vec.get_feature_names()))<jupyter_output><empty_output><jupyter_text>TfidfVectorizer... https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html<jupyter_code>sample = ['problem of evil',
          'evil queen',
          'horizon problem']
from sklearn.feature_extraction.text import TfidfVectorizer
vec = TfidfVectorizer()

X = vec.fit_transform(sample)
pd.DataFrame(X.toarray(), columns=vec.get_feature_names())<jupyter_output><empty_output><jupyter_text><jupyter_code>sample = ['problem of evil',
          'problem of evil',
          'evil queen',
          'horizon problem',
          'winston of']

vec = TfidfVectorizer()

X = vec.fit_transform(sample)

pd.DataFrame(X.toarray(), columns=vec.get_feature_names())
sample = ['problem of evil',
          'problem of evil',
          'evil queen',
          'horizon problem',
          'winston of of of ']

vec = TfidfVectorizer()

X = vec.fit_transform(sample)

pd.DataFrame(X.toarray(), columns=vec.get_feature_names())
sample = ['problem of evil',
          'problem of evil',
          'evil queen',
          'horizon problem',
          'winston of of of ',
          'winston']

vec = TfidfVectorizer()

X = vec.fit_transform(sample)

pd.DataFrame(X.toarray(), columns=vec.get_feature_names())<jupyter_output><empty_output><jupyter_text>![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZMAAAENCAYAAADKcIhSAAAgAElEQVR4Ae3dB6xsSXE3cGzL7BphMAYWxBKECSKKKJYgopYkTPhMFDYmoyUZHiAMrIAFEYyIIllEEY1JIgqMESaIKKKIIsq7gIgiyjZg8Hz6td//ubeZdO/Mve/O3GrpTJ/Tobq6urqquvpMn3OcccYZJ7rOOuusyTe/+c3Jv//7v9dVNCgeKB4oHigeWMgD3/jGNyb0xz3ucY8Tz5Gb73//+5Nf//rXk9/+92/rKhoUDxQPFA<KEY>) Tfidf on airbnb data...<jupyter_code>df['title']
sample = df['title']

vec = TfidfVectorizer()

X = vec.fit_transform(sample)

pd.DataFrame(X.toarray(), columns=vec.get_feature_names())<jupyter_output><empty_output><jupyter_text>Derived Features<jupyter_code>df
t = pd.DataFrame()

t['price_per_bed'] = df['beds'] / df['price']

t
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

x = np.array([1, 2, 3, 4, 5])
y = np.array([4, 2, 1, 3, 7])
plt.scatter(x, y);
X = x[:, np.newaxis]
X.shape
x.shape
y
from sklearn.linear_model import LinearRegression

X = x[:, np.newaxis]

# y = mx + b
model = LinearRegression().fit(X, y)
yfit = model.predict(X)

plt.scatter(x, y)
plt.plot(x, yfit);<jupyter_output><empty_output><jupyter_text>https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.PolynomialFeatures.html<jupyter_code>X
from sklearn.preprocessing import PolynomialFeatures

poly = PolynomialFeatures(degree=3, include_bias=False)

X2 = poly.fit_transform(X)

print(X2)
from sklearn.preprocessing import PolynomialFeatures

poly = PolynomialFeatures(degree=2, include_bias=False)

X2 = poly.fit_transform(np.array([1, 2]).reshape(-1, 1))

print(X2)
X2 = poly.fit_transform(np.array([[1, 2], [1, 3]]))
X2

from sklearn.preprocessing import PolynomialFeatures

poly = PolynomialFeatures(degree=3, include_bias=False)
X2 = poly.fit_transform(X)

print(X2)
model = LinearRegression().fit(X2, y)

yfit = model.predict(X2)

plt.scatter(x, y)
plt.plot(x, yfit);<jupyter_output><empty_output><jupyter_text>Missing Data<jupyter_code>import numpy as np 

X = np.array([[np.nan, 0,   3  ],
              [ 3,   7,   9  ],
              [ 3,   5,   2  ],
              [ 4,   np.nan, 6  ],
              [ 8,   8,   1  ]])

y = np.array([14, 16, -1,  8, -5])
pd.DataFrame(X)
pd.DataFrame(X).fillna(0)
pd.DataFrame(X).fillna('a string')<jupyter_output><empty_output><jupyter_text>- https://scikit-learn.org/stable/modules/generated/sklearn.impute.SimpleImputer.html
- https://scikit-learn.org/stable/modules/impute.html<jupyter_code>X
from sklearn.impute import SimpleImputer

imp = SimpleImputer(missing_values=np.nan, strategy='mean')

X2 = imp.fit_transform(X)

X2
model = LinearRegression().fit(X2, y)
model.predict(X2)<jupyter_output><empty_output><jupyter_text>If that simple... could also just use pandas..<jupyter_code>t = pd.DataFrame(X)

t.fillna(t.mean())<jupyter_output><empty_output><jupyter_text>Random Test<jupyter_code>import pandas as pd

tokyo = 'https://gist.githubusercontent.com/gumdropsteve/40555dbb4cb01e694328879175e5feaf/raw/59a74ffbb4000a37f1bdb2f61dc7e702e07fa6e7/tokyo.csv'

df = pd.read_csv(tokyo)

df.tail(3)
from sklearn.feature_extraction import DictVectorizer

vec = DictVectorizer(sparse=False, dtype=int)

# get list of dicts from `type` column
data = [{'type':v} for v in df.type]

d = vec.fit_transform(data)

type_df = pd.DataFrame(d, columns=vec.feature_names_)
type_df.tail(3)
sample = df['title']

vec = TfidfVectorizer()

X = vec.fit_transform(sample)

tfidf_df = pd.DataFrame(X.toarray(), columns=vec.get_feature_names())
tfidf_df.tail(3)
big_df = pd.concat([df, type_df, tfidf_df], axis=1)

big_df
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split

X = df.drop(['price', 'ds'], axis=1)
y = df.price

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, random_state=7)
lr = LinearRegression()

lr.fit(X_train, y_train)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14555862209956047, 'fraction_numerical': 0.02657671470918941, 'mean_word_length': 4.862192929898143, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 70, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '253167', 'license': 'MIT', 'n_tokens_mistral': 4045, 'n_tokens_neox': 3758, 'n_words': 634, 'path': 'stack_4/day_04/01_feature_extraction.ipynb', 'repo_name': 'mpHarm88/learn_datascience', 'size': '47884'}