<jupyter_start><jupyter_text>Load data<jupyter_code>raw_data_train = pd.read_csv('../data/raw/train.csv')
raw_data_test = pd.read_csv('../data/raw/test.csv')
target_train = raw_data_train['y']
raw_data_train.drop('y', axis=1, inplace=True)
feature_groups = [
    [0, 1],
    [44, 43, 46, 62, 45, 13, 77, 61, 58, 4, 55, 56, 5, 75],
    [28, 26, 38, 37, 30, 31, 33, 34, 27, 29, 32, 36],
    [14, 23, 19, 20, 15, 24, 18],
    [2, 3, 17, 16, 39, 40, 21, 22, 25, 53, 69, 11, 12, 41, 42, 66, 76],
    [8, 70, 71, 10, 73, 74, 9, 6, 51, 52, 7, 49, 50, 63, 59, 60, 65],
    [57, 67, 68],
    [35, 48, 54],
    [72, 78, 47],
    [64],
]<jupyter_output><empty_output><jupyter_text>**Explore feature groups**<jupyter_code>feature_group_index = 1
raw_data_train.iloc[:, feature_groups[feature_group_index]].describe(include='all').T<jupyter_output><empty_output><jupyter_text>Build features<jupyter_code>from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OrdinalEncoder, OneHotEncoder, FunctionTransformer
from sklearn.impute import SimpleImputer

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
initial_preprocessor = ColumnTransformer(
    [
        ('year', OrdinalEncoder(dtype='int'), ['galactic year']),
        
        ('galaxy', FeatureUnion(
            [
                ('onehot', OneHotEncoder(dtype='int')),
                ('bagofwords', Pipeline(
                    [
                        ('unravel', FunctionTransformer(lambda df: df.iloc[:, 0])),
                        ('vectorize', CountVectorizer(lowercase=True, binary=True, token_pattern=r'(?u)\b\S+\b')),
                    ])
                ),
            ],
            n_jobs=-1
        ), ['galaxy']),
        
    ],
    remainder='passthrough',
    n_jobs=-1,
    verbose=True,
    sparse_threshold=0
)
train_data_size = raw_data_train.shape[0]
num_original_features = raw_data_train.shape[1] - len(initial_preprocessor.transformers)
interim_features = initial_preprocessor.fit_transform(pd.concat([raw_data_train, raw_data_test], sort=False))

interim_features_train = interim_features[:train_data_size]
interim_features_test = interim_features[train_data_size:]
preprocessor = ColumnTransformer(
    [
        ('general', Pipeline(
            [
                ('imputer', IterativeImputer(initial_strategy='mean', imputation_order='ascending', random_state=RANDOM_STATE)),
                ('scaler', StandardScaler()),
            ],

            memory='../data/interim/',
            verbose=True,
        ), slice(-num_original_features, None))
    ],
    remainder='passthrough',
    n_jobs=-1,
    verbose=True
)
features_train = preprocessor.fit_transform(interim_features_train)
features_test = preprocessor.fit_transform(interim_features_test)<jupyter_output><empty_output><jupyter_text>Train models<jupyter_code>from sklearn.feature_selection import SelectKBest, f_regression, mutual_info_regression
from sklearn.model_selection import GridSearchCV

from sklearn.metrics import mean_squared_error, make_scorer

scoring_func = make_scorer(mean_squared_error, greater_is_better=False, squared=False)<jupyter_output><empty_output><jupyter_text>Universal model<jupyter_code>from sklearn.ensemble import GradientBoostingRegressor
estimator = Pipeline(
    [
        ('preprocessor', preprocessor),
        #('feature_selector', SelectKBest(mutual_info_regression)),
        ('estimator', GradientBoostingRegressor(n_estimators=300, random_state=RANDOM_STATE)),
    ],
    verbose=True
)
grid_search = GridSearchCV(
    estimator, 
    param_grid={
        #'feature_selector__k': range(10, 490, 10),
    }, 
    scoring=scoring_func,
    cv=5,
    n_jobs=-1, 
    verbose=6,
)
grid_search.fit(features_train, target_train)
print(grid_search.best_score_)<jupyter_output>-0.032561213580010996<jupyter_text>Model per galaxy<jupyter_code>models = {}
for i, galaxy_name in enumerate(raw_data_train['galaxy'].unique()):
    galaxy_index = (raw_data_train['galaxy'] == galaxy_name)
    galaxy_features = features_train[galaxy_index]
    galaxy_target = target_train[galaxy_index]
    models[galaxy_name] = GradientBoostingRegressor(n_estimators=100, random_state=RANDOM_STATE).fit(galaxy_features, galaxy_target)<jupyter_output><empty_output><jupyter_text>Generate predictions  Universal model<jupyter_code>predictions_train = grid_search.best_estimator_.predict(features_train)
mean_squared_error(target_train, predictions_train, squared=False)<jupyter_output><empty_output><jupyter_text>First try: 0.13000 After pipelining and bow: 0.06440 After iterative imputation: 0.01858  Model per galaxy<jupyter_code>predictions_train = np.empty(features_train.shape[0])
for i, galaxy_name in raw_data_train['galaxy'].iteritems():
    predictions_train[i] = models[galaxy_name].predict(features_train[i, :].reshape(1, -1))
mean_squared_error(target_train, predictions_train, squared=False)<jupyter_output><empty_output><jupyter_text>First try: 0.00590 After pipelining and bow: 0.00030 After iterative imputation: no significant change<jupyter_code>predictions_test = np.empty(features_test.shape[0])
for i, galaxy_name in raw_data_test['galaxy'].iteritems():
    predictions_test[i] = models[galaxy_name].predict(features_test[i, :].reshape(1, -1))<jupyter_output><empty_output><jupyter_text>Optimization problem<jupyter_code>energy_allocations = raw_data_test[['galaxy', 'galactic year','existence expectancy index']]<jupyter_output><empty_output><jupyter_text>Identify galaxies with top potential for increase in the index<jupyter_code>index_increase_potential = (-np.log(predictions_test+0.01)+3)**2 / 1000
plt.hist(index_increase_potential)
top_indices = np.argsort(index_increase_potential)[-500:]<jupyter_output><empty_output><jupyter_text>Allocate accordingly<jupyter_code>TOTAL_AVAILABLE_ENERGY = 50000
energy_allocations['share'] = np.zeros(energy_allocations.shape[0])
energy_allocations.loc[top_indices, 'share'] = 100
energy_allocations['share'].value_counts()<jupyter_output><empty_output><jupyter_text>**Check allocations for validity**<jupyter_code>GALAXY_ENERGY_MAX = 100
(energy_allocations['share'] > GALAXY_ENERGY_MAX).any()
GALAXY_ENERGY_MIN = 0
(energy_allocations['share'] < GALAXY_ENERGY_MIN).any()
energy_allocations['share'].sum() > TOTAL_AVAILABLE_ENERGY
EEI_THRESHOLD = 0.7
SUBSIDIZED_ENERGY_SHARE = 0.1

subsidized_galaxies = energy_allocations['existence expectancy index'] < EEI_THRESHOLD
energy_allocations.loc[subsidized_galaxies, ['share']].sum()['share'] < SUBSIDIZED_ENERGY_SHARE * TOTAL_AVAILABLE_ENERGY<jupyter_output><empty_output><jupyter_text>Export predictions<jupyter_code>submitions_sample = pd.read_csv('../submitions/sample_submit.csv')
submitions = submitions_sample.copy(deep=True)

submitions['pred'] = predictions_test
submitions['opt_pred'] = energy_allocations['share']
submitions.to_csv(f'../submitions/submitions {datetime.datetime.now():%Y-%m-%dT%H-%M-%S}.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1251226005324366, 'fraction_numerical': 0.0360095278128065, 'mean_word_length': 5.702347417840375, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '586572', 'license': 'MIT', 'n_tokens_mistral': 2721, 'n_tokens_neox': 2442, 'n_words': 376, 'path': 'notebooks/2020-06 McK Prohack.ipynb', 'repo_name': 'vladimir-alekseev/ml_comps-mckinsey-prohack-2020', 'size': '7137'}