<jupyter_start><jupyter_text><jupyter_code>!pip install git+https://github.com/fastai/fastai@2e1ccb58121dc648751e2109fc0fbf6925aa8887
!apt update && apt install -y libsm6 libxext6
#!pip install fastai
!pip install pandas-summary
!pip install kaggle-cli
!kg download -u 'user' -p 'password' -c bluebook-for-bulldozers -f Train.zip
%load_ext autoreload
%autoreload 2
%matplotlib inline
from fastai.imports import *

from fastai.structured import *
from pandas_summary import DataFrameSummary

from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display

from sklearn import metrics
!curl -O https://raw.githubusercontent.com/agustincl/aisaturdays/master/data/Train.zip
!mkdir -p data/bulldozers/
!mv Train.zip data/bulldozers/
!unzip data/bulldozers/Train.zip -d data/bulldozers/
!mv bluebook-for-bulldozers.zip data/bulldozers/
!rm -rf data/bulldozers/Train.zip
!mv Train.zip data/bulldozers/
!unzip data/bulldozers/Train.zip -d data/bulldozers/
!ls -la data/bulldozers/
PATH = "data/bulldozers/"
df_raw = pd.read_csv(f'{PATH}Train.csv', low_memory=False, parse_dates=["saledate"])
df_raw.info()
df_raw.SalePrice = np.log(df_raw.SalePrice)
def add_datepart(df, fldname, drop=True):
    fld = df[fldname]
    if not np.issubdtype(fld.dtype, np.datetime64):
        df[fldname] = fld = pd.to_datetime(fld, 
                                     infer_datetime_format=True)
    targ_pre = re.sub('[Dd]ate$', '', fldname)
    for n in ('Year', 'Month', 'Week', 'Day', 'Dayofweek', 
            'Dayofyear', 'Is_month_end', 'Is_month_start', 
            'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 
            'Is_year_start'):
        df[targ_pre+n] = getattr(fld.dt,n.lower())
    df[targ_pre+'Elapsed'] = fld.astype(np.int64) // 10**9
    if drop: df.drop(fldname, axis=1, inplace=True)
def display_all(df):
    with pd.option_context("display.max_rows", 1000): 
        with pd.option_context("display.max_columns", 1000): 
            display(df)
add_datepart(df_raw, 'saledate')
display_all(df_raw.tail().transpose())
df_raw.SalePrice = np.log(df_raw.SalePrice)
df_raw.columns
train_cats(df_raw)
df_raw.UsageBand.cat.categories
df_raw.UsageBand.cat.set_categories(['High', 'Medium', 'Low'], ordered=True, inplace=True)
df, y, nas = proc_df(df_raw, 'SalePrice')
m = RandomForestRegressor(n_jobs=-1)
m.fit(df, y)
m.score(df,y)
def split_vals(a,n): return a[:n].copy(), a[n:].copy()
n_valid = 12000  # same as Kaggle's test set size
n_trn = len(df)-n_valid
raw_train, raw_valid = split_vals(df_raw, n_trn)
X_train, X_valid = split_vals(df, n_trn)
y_train, y_valid = split_vals(y, n_trn)
X_train.shape, y_train.shape, X_valid.shape
def rmse(x,y): return math.sqrt(((x-y)**2).mean())
def print_score(m):
    res = [rmse(m.predict(X_train), y_train),        
                rmse(m.predict(X_valid), y_valid),
                m.score(X_train, y_train), m.score(X_valid,   
                y_valid)]
    if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
    print(res)
m = RandomForestRegressor(n_jobs=-1)
%time m.fit(X_train, y_train)
print_score(m)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14181007010834926, 'fraction_numerical': 0.016889738687061822, 'mean_word_length': 4.33843537414966, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '580946', 'license': 'MIT', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1225, 'n_words': 226, 'path': 'ML_Lesson1.ipynb', 'repo_name': 'agustincl/aisaturdays', 'size': '3138'}