<jupyter_start><jupyter_text>Read in data<jupyter_code>print('Reading in train.csv')
train_data = pd.read_csv(path/'train.csv')
print('Reading in train_labels.csv')
train_labels = pd.read_csv(path/'train_labels.csv')
print('Reading in test.csv')
test_data = pd.read_csv(path/'test.csv')
print('Reading in specs.csv')
specs = pd.read_csv(path/'specs.csv')<jupyter_output><empty_output><jupyter_text>Reduce the size of the data frames<jupyter_code>## Function to reduce the DF size
def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage().sum() / 1024**2    
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)    
    end_mem = df.memory_usage().sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    return df
train_data = reduce_mem_usage(train_data)
train_labels = reduce_mem_usage(train_labels)
test_data = reduce_mem_usage(test_data)
specs = reduce_mem_usage(specs)
# helper function to decompose dates into more relevant parts
def explode_date(col):
    dayofyear = col.dt.dayofyear
    weekofyear = col.dt.weekofyear
    weekday = col.dt.weekday
    month = col.dt.month
    year = col.dt.year
    hour = col.dt.hour
    quarter = col.dt.quarter
    return pd.DataFrame({'dayofyear':dayofyear,
                         'weekofyear':weekofyear,
                         'weekday':weekday,
                         'quarter':quarter,
                         'month':month,
                         'year':year,
                         'hour':hour})<jupyter_output><empty_output><jupyter_text>Main funtion that aggregates the sequence data<jupyter_code># reduces data with respect to each installation id
def reduce_data_per_user(user_dataframe):
    event_counts = {'total_events':0,'clips_count':0,'assessment_count':0,'activity_count':0,'game_count':0,
                    'correct_events_count':0,'incorrect_events_count':0}
    specific_event_counts = dict(zip(specs.event_id,[0 for _ in range(specs.shape[0])]))
    df_features = []
    for idx,session in user_dataframe.groupby('game_session'):
        if session.type.iloc[0]=='Assessment':
            assessment_info = {'assess_world':session.world.iloc[0],
                               'assess_title':session.title.iloc[0],
                               'game_session':session.game_session.iloc[0],
                               'installation_id':session.installation_id.iloc[0]
                              }
            df_features.append({**event_counts,**assessment_info,**specific_event_counts})
            event_counts['assessment_count'] += (session.type == 'Assessment').sum()
        else:
            event_counts['total_events'] += session.event_id.count()
            event_counts['clips_count'] += (session.type == 'Clip').sum()
            event_counts['activity_count'] += (session.type == 'Activity').sum()
            event_counts['game_count'] += (session.type == 'Game').sum()
            event_counts['correct_events_count'] += session.event_id.str.contains('Correct').sum()
            event_counts['incorrect_events_count'] += session.event_id.str.contains('Incorrect').sum()
            specific_event_counts.update(dict(session.event_id.value_counts())) 
    return df_features
def reduce_data(install_id,data,is_train_data=True):
    rows = []
    for idx,df in tqdm(data.groupby('installation_id')):
        rows.extend(reduce_data_per_user(df))
    data = pd.DataFrame(rows)
    if is_train_data:
        return train_labels[['game_session','accuracy_group']]\
                        .merge(data,on='game_session',how='inner')\
                        .drop(['game_session'],axis=1)
    else:
        return data.groupby('installation_id',as_index=False).apply(lambda x: x.iloc[-1]).drop(['game_session'],axis=1)

train_reduced = reduce_data(train_labels.installation_id.unique(),train_data)
test_reduced = reduce_data(test_data.installation_id.unique(),test_data, is_train_data = False)<jupyter_output><empty_output><jupyter_text>Modeling<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import cohen_kappa_score,confusion_matrix
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import train_test_split
worlds = list(set(train_reduced.assess_world.unique()).union(test_reduced.assess_world.unique()))
titles = list(set(train_reduced.assess_title.unique()).union(test_reduced.assess_title.unique()))
train,val = train_test_split(train_reduced,test_size=.2)
X_col_names = [col for col in train_reduced.columns if col != 'accuracy_group']
train_x = train.loc[:,X_col_names]
train_y = train.loc[:,'accuracy_group']
val_x = val.loc[:,X_col_names]
val_y = val.loc[:,'accuracy_group']
preprocessor = ColumnTransformer([('onehot1',OneHotEncoder(),['assess_world']),
                                  ('onehot2',OneHotEncoder(),['assess_title']),
                                  ('drop', 'drop',['installation_id'])],
                                  remainder='passthrough')
pipe = Pipeline([('col_transform',preprocessor),
                 ('cls',RandomForestClassifier())])
pipe.fit(train_x,train_y)
preds = pipe.predict(val_x)
cohen_kappa_score(val_y,preds,weights='quadratic')
def prepare_submission(model,test_data):
    test_predictions = model.predict(test_data)
    submission_df = pd.DataFrame({'installation_id':test_data.installation_id.to_numpy(),
                                  'accuracy_group':test_predictions})
    return submission_df
test = prepare_submission(pipe,test_reduced)
test.to_csv('submission.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1335052284772955, 'fraction_numerical': 0.012032660077352814, 'mean_word_length': 3.1264775413711585, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '226154', 'license': 'MIT', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2246, 'n_words': 337, 'path': 'submissions/submission1.ipynb', 'repo_name': 'loftiskg/kaggle-DS-bowl2019', 'size': '6981'}