<jupyter_start><jupyter_text>Lambda School Data Science*Unit 2, Sprint 3, Module 1*--- Wrangle ML datasets- [ ] Continue to clean and explore your data. - [ ] For the evaluation metric you chose, what score would you get just by guessing?- [ ] Can you make a fast, first model that beats guessing?**We recommend that you use your portfolio project dataset for all assignments this sprint.****But if you aren't ready yet, or you want more practice, then use the New York City property sales dataset for today's assignment.** Follow the instructions below, to just keep a subset for the Tribeca neighborhood, and remove outliers or dirty data. [Here's a video walkthrough](https://youtu.be/pPWFw8UtBVg?t=584) you can refer to if you get stuck or want hints!- Data Source: [NYC OpenData: NYC Citywide Rolling Calendar Sales](https://data.cityofnewyork.us/dataset/NYC-Citywide-Rolling-Calendar-Sales/usep-8jbt)- Glossary: [NYC Department of Finance: Rolling Sales Data](https://www1.nyc.gov/site/finance/taxes/property-rolling-sales-data.page) Your code starts here:<jupyter_code>!wget 'https://raw.githubusercontent.com/washingtonpost/data-school-shootings/master/school-shootings-data.csv'
import pandas as pd

df = pd.read_csv('school-shootings-data.csv')
print(df.shape)
df.head()
# Replace shooting type with 'other' for rows not 'targeted' or 'indiscriminate'
  df['shooting_type'] = df['shooting_type'].replace(['accidental', 'unclear',
                                                    'targeted and indiscriminate',
                                                    'public suicide',
                                                    'hostage suicide',
                                                    'accidental or targeted',
                                                    'public suicide (attempted)'],
                                                    'other')

# Fill missing value with 'other'
  df['shooting_type'] = df['shooting_type'].fillna('other')
# Majority class baseline 59%
df['shooting_type'].value_counts(normalize=True)
from sklearn.model_selection import train_test_split

# Create train, test
train, test = train_test_split(df, train_size=0.80, random_state=21, stratify=df['shooting_type'])

train.shape, test.shape
def wrangle(df):

  # Avoid SettingWithCopyWarning
  df = df.copy()

  # Remove commas from numbers
  df['white'] = df['white'].str.replace(",", "")

  # Change from object to int
  df['white'] = pd.to_numeric(df['white'])
  
  # Remove commas from numbers
  df['enrollment'] = df['enrollment'].str.replace(",", "")

  # Change from object to int
  df['enrollment'] = pd.to_numeric(df['enrollment'])

  # Fill missing values for these specific columns
  df.fillna({'white': 0, 'black': 0, 'hispanic': 0, 'asian': 0,
           'american_indian_alaska_native': 0,
           'hawaiian_native_pacific_islander': 0, 'two_or_more': 0,
           'district_name': 'Unknown', 'time': '12:00 PM', 'lat': 33.612910,
           'long': -86.682000, 'staffing': 60.42, 'low_grade': '9',
           'high_grade': '12'}, inplace=True)
  
  # Drop columns with 200+ missing values
  df = df.drop(columns=['deceased_notes1', 'age_shooter2', 'gender_shooter2', 
                        'race_ethnicity_shooter2', 'shooter_relationship2', 
                        'shooter_deceased2', 'deceased_notes2'])

  # Drop unusable variance 
  df = df.drop(columns=['uid', 'nces_school_id', 'nces_district_id', 'weapon', 
                        'weapon_source', 'state_fips', 'county_fips', 'ulocale',
                        'lunch', 'age_shooter1', 'gender_shooter1',
                        'race_ethnicity_shooter1', 'shooter_relationship1',
                        'shooter_deceased1'])
  
  # Change date to datettime
  df['date'] = pd.to_datetime(df['date'])

  return df

train = wrangle(train)
test = wrangle(test)
train.shape, test.shape
!pip install category_encoders==2.*
import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import f_classif, SelectKBest
from sklearn.linear_model import Ridge

target = 'shooting_type'
features = train.columns.drop([target, 'date'])
X_train = train[features]
y_train = train[target]
X_test = test[features]
y_test = test[target]

pipeline = make_pipeline(
    ce.OrdinalEncoder(), 
    StandardScaler(), 
    RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=21)
)

k = 20
scores = cross_val_score(pipeline, X_train, y_train, cv=k)
print(f'MAE for {k} folds:', scores)
scores.mean()
from sklearn.tree import DecisionTreeClassifier

target = 'shooting_type'
features = train.columns.drop([target, 'date', ])
X_train = train[features]
y_train = train[target]
X_test = test[features]
y_test = test[target]

pipeline = make_pipeline(
    ce.OrdinalEncoder(), 
    DecisionTreeClassifier(max_depth=3)
)

pipeline.fit(X_train, y_train)
print('Test Accuracy:', pipeline.score(X_test, y_test))<jupyter_output>Test Accuracy: 0.5416666666666666
{'fraction_non_alphanumeric': 0.11666988603438284, 'fraction_numerical': 0.01777090979331659, 'mean_word_length': 3.4068085106382977, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '829', 'license': 'MIT', 'n_tokens_mistral': 1677, 'n_tokens_neox': 1557, 'n_words': 410, 'path': 'module2-wrangle-ml-datasets/LS_DS12_232_assignment.ipynb', 'repo_name': 'jdz014/DS-Unit-2-Applied-Modeling', 'size': '5177'}