<jupyter_start><jupyter_text>Coal production in mines 2013by: <NAME>Abstract: We did alot of analysis, came to good conclusion.<jupyter_code>%matplotlib inline

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import explained_variance_score, r2_score, mean_squared_error

sns.set();<jupyter_output>/anaconda/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)<jupyter_text>Cleaned DataWe cleaned this data in the notebook stored in: deliver/Data_cleaning.ipynb<jupyter_code>df1 = pd.read_csv("../data/cleaned_coalpublic2013.csv", index_col='MSHA ID')
df1[['year', 'mine_name']].head()<jupyter_output><empty_output><jupyter_text>Predict the Production of coal mines<jupyter_code>df1['log_production'] = np.log(df1['production_(short_tons)'])
df1.union_code.unique()
features = ['average_employees',
            'labor_hours']

categoricals = ['mine_state',
                'mine_county',
                'mine_status',
                'mine_type',
                'company_type',
                'operation_type',
                'operating_company',
                'union_code',
                'coal_supply_region']

target = 'log_production'
sns.set_context('poster')
fig = plt.subplots(figsize=(14,8))
sns.violinplot(y="company_type", x="log_production", data=df1,
              split=True, inner="stick")
plt.tight_layout()
plt.savefig("../figures/Coal_prediction_company_type_vs_log_production.png")
dummy_categoricals = []
for categorical in categoricals:
    # Avoid the dummy variable trap (need to remove one)!
    drop_var = sorted(df1[categorical].unique())[-1]
    temp_df = pd.get_dummies(df1[categorical], prefix=categorical)
    df1 = pd.concat([df1, temp_df], axis=1)
    temp_df.drop('_'.join([categorical, str(drop_var)]), axis=1, inplace=True)
    dummy_categoricals += temp_df.columns.tolist()<jupyter_output><empty_output><jupyter_text>Random Forest Regressor<jupyter_code>train, test = train_test_split(df1, test_size=0.3)
rf = RandomForestRegressor(n_estimators=100, oob_score=True)
rf.fit(train[features + dummy_categoricals], train[target])
fig = plt.subplots(figsize=(8,8))
sns.regplot(test[target], rf.predict(test[features + dummy_categoricals]))
plt.ylabel("Predicted Production")
plt.xlim(0, 22)
plt.ylim(0, 22)
plt.tight_layout()
plt.savefig("../figures/Coal_prediction_rf_predition.png")
predicted = rf.predict(test[features + dummy_categoricals])
print "R^2 score: ", r2_score(test[target], predicted)

print "MSE: ", mean_squared_error(test[target], predicted)
rf_importances = pd.DataFrame({'name':train[features + dummy_categoricals].columns,
                              'importance':rf.feature_importances_
                              }).sort_values(by='importance', ascending=False).reset_index(drop=True)
rf_importances.head(5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11687144141444411, 'fraction_numerical': 0.01648186994306263, 'mean_word_length': 4.907964601769912, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '493385', 'license': 'MIT', 'n_tokens_mistral': 1143, 'n_tokens_neox': 1091, 'n_words': 222, 'path': 'deliver/Coal_prediction.ipynb', 'repo_name': 'wolfe-man/coal-exploration', 'size': '3344'}