<jupyter_start><jupyter_text>Tanzania Water Pumps  Imports<jupyter_code># !pip install -U pandas-profiling
!pip install category-encoders
# libraries
import pandas as pd
import pandas_profiling
import numpy as np

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

import category_encoders as ce
# Data
LOCAL = '../data/tanzania/'
WEB = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Linear-Models/master/data/tanzania/'

train_features = pd.read_csv(WEB + 'train_features.csv')
train_labels = pd.read_csv(WEB + 'train_labels.csv')
test_features = pd.read_csv(WEB + 'test_features.csv')
sample_submission = pd.read_csv(WEB + 'sample_submission.csv')

assert train_features.shape == (59400, 40)
assert train_labels.shape == (59400, 2)
assert test_features.shape == (14358, 40)
assert sample_submission.shape == (14358, 2)<jupyter_output><empty_output><jupyter_text>Data Exploration<jupyter_code>train_features.head()<jupyter_output><empty_output><jupyter_text>Majority Class Baseline<jupyter_code>y_train = train_labels['status_group']
y_train.value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>`functional` is the majority class.<jupyter_code># Our baseline model
majority_class = y_train.mode()[0]
y_pred = [majority_class] * len(y_train)
# Check the accuracy score
accuracy_score(y_train, y_pred)<jupyter_output><empty_output><jupyter_text>Train/Validate/Test Split *RUN THIS TO RESET TRAINING AND VALIDATION SETS*<jupyter_code># Train/Test already Split
X_train = train_features
y_train = train_labels['status_group']

# Validation Split
X_train, X_val, y_train, y_val = train_test_split(X_train,
                                                  y_train,
                                                  train_size=0.8,
                                                  test_size=0.2,
                                                  stratify=y_train,
                                                  random_state=66)

# Check the shape
X_train.shape, X_val.shape, y_train.shape, y_val.shape
# Doublecheck Stratification
assert(y_train.value_counts(normalize=True)[0] - y_val.value_counts(normalize=True)[0] < 0.01)
print('Stratification is fine')<jupyter_output>Stratification is fine<jupyter_text>Basic Logistic Regression<jupyter_code># Only take numeric data
X_train_numeric = X_train.select_dtypes('number')
X_val_numeric = X_val.select_dtypes('number')
# Instantiate model
model = LogisticRegression(solver='lbfgs',
                         multi_class='auto',
                         max_iter=10000)

# Fit model
model.fit(X_train_numeric, y_train)
# Make predictions
y_pred = model.predict(X_val_numeric)

# Test the accuracy of the predictions
accuracy_score(y_val, y_pred)<jupyter_output><empty_output><jupyter_text>Logistic Regression* Numeric features except `id`* OneHotEncode `quantity`<jupyter_code># Train/Test already Split
X_train = train_features
y_train = train_labels['status_group']

# Validation Split
X_train, X_val, y_train, y_val = train_test_split(X_train,
                                                  y_train,
                                                  train_size=0.8,
                                                  test_size=0.2,
                                                  stratify=y_train,
                                                  random_state=66)

# Check the shape
X_train.shape, X_val.shape, y_train.shape, y_val.shape
# Find all the numeric features
X_train_numeric = X_train.select_dtypes(include='number')
numeric_features = list(X_train_numeric.columns.values)
numeric_features.remove('id')

# Add quantity
features = numeric_features
features.append('source')

features
# Train/Test already Split
X_train = train_features[features]
y_train = train_labels['status_group']

# Remove features we don't want from X_val
X_val = X_val[features]

# OneHotEncode `quantity`
encoder = ce.OneHotEncoder(use_cat_names=True)
X_train = encoder.fit_transform(X_train)
X_val = encoder.transform(X_val)

# Scale the data (Unit Normal)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_val = scaler.transform(X_val)
# Instantiate model
model = LogisticRegression(solver='lbfgs',
                         multi_class='auto',
                         max_iter=10000)

# Fit model
model.fit(X_train, y_train)
# Make predictions
y_pred = model.predict(X_val)

# Test the accuracy of the predictions
accuracy_score(y_val, y_pred)<jupyter_output><empty_output><jupyter_text>Logistic Regression* Numeric features except `id`* OneHotEncode ``<jupyter_code># Train/Test already Split
X_train = train_features
y_train = train_labels['status_group']

# Validation Split
X_train, X_val, y_train, y_val = train_test_split(X_train,
                                                  y_train,
                                                  train_size=0.8,
                                                  test_size=0.2,
                                                  stratify=y_train,
                                                  random_state=66)

# Check the shape
X_train.shape, X_val.shape, y_train.shape, y_val.shape
# Find all the numeric features
X_train_numeric = X_train.select_dtypes(include='number')
numeric_features = list(X_train_numeric.columns.values)
numeric_features.remove('id')
numeric_features.remove('num_private')

# Add features to be encoded
features = numeric_features
features.append('quantity')
features.append('installer')
features.append('extraction_type')
features.append('source')

# features
# Train/Test already Split
X_train = train_features[features]
y_train = train_labels['status_group']

# Remove features we don't want from X_val
X_val = X_val[features]

# OneHotEncode `quantity`
encoder = ce.OneHotEncoder(use_cat_names=True)
X_train = encoder.fit_transform(X_train)
X_val = encoder.transform(X_val)

# Scale the data (Unit Normal)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_val = scaler.transform(X_val)
# Instantiate model
model = LogisticRegression(solver='lbfgs',
                         multi_class='auto',
                         max_iter=10000)

# Fit model
model.fit(X_train, y_train)
# Make predictions
y_pred = model.predict(X_val)

# Test the accuracy of the predictions
accuracy_score(y_val, y_pred)
X_val.shape<jupyter_output><empty_output><jupyter_text>Export a Submission<jupyter_code># Create & process y_test
X_test = test_features[features]

# Encode
X_test = encoder.transform(X_test)

# Scale to Unit Normal
X_test = scaler.transform(X_test)
# Make predictions
y_pred = model.predict(X_test)
# Send predictions to df
submission_df = pd.DataFrame()

# Add columns
submission_df['id'] = True
submission_df['id'] = test_features['id'].tolist()

submission_df['status_group'] = True
submission_df['status_group'] = y_pred

submission_df.head()
submission_df.to_csv('submission', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09308586001700199, 'fraction_numerical': 0.009351090960612072, 'mean_word_length': 3.445214105793451, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '141714', 'license': 'MIT', 'n_tokens_mistral': 2265, 'n_tokens_neox': 2166, 'n_words': 434, 'path': 'LogisticRegression_EricWuerfel.ipynb', 'repo_name': 'ewuerfel66/DS-Unit-2-Linear-Models', 'size': '7058'}