<jupyter_start><jupyter_text>Group Count + Truncated SVD<jupyter_code>### CREATE ENCODING FOR EACH CATEGORICAL FEATURES ###

diz_enc = {}

for g_f in features:
    
    passive_feat = features[:]
    passive_feat.remove(g_f)
    
    diz_group = defaultdict(list)
    for f in passive_feat:
        
        print('--- groupby:', g_f, '###', f, '---')
        group = df_train[[g_f,f]].astype(str).groupby(g_f)[f]
        group = group.apply(lambda x: ' '.join(list(x)))
        group_index = ['0']+list(group.index)
        
        enc_count = CountVectorizer()
        group = enc_count.fit_transform(group)
        group = sparse.vstack((sparse.csr_matrix((1,group.shape[1]), dtype='int64'), group))
        
        LSA = TruncatedSVD(n_components=3, random_state=33)
        group_enc = LSA.fit_transform(group)
        
        for i,k in enumerate(group_index):
            diz_group[int(k)].extend(list(group_enc[i]))
    
    diz_enc[g_f] = diz_group
### MAP CLASSES TO VECTORS ###

train_feat = df_train.copy()
val_feat = df_val.copy()
test_feat = df_test.copy()

for f in features:
    train_feat[f] = train_feat[f].map(diz_enc[f])
    val_feat[f] = val_feat[f].map(diz_enc[f])
    test_feat[f] = test_feat[f].map(diz_enc[f])
    
train_feat = [list(itertools.chain(*raw[1:])) for raw in train_feat[features].itertuples()]
val_feat = [list(itertools.chain(*raw[1:])) for raw in val_feat[features].itertuples()]
test_feat = [list(itertools.chain(*raw[1:])) for raw in test_feat[features].itertuples()]
### GET LABELS ###

y_train = df_train.is_attributed.values
y_val = df_val.is_attributed.values
y_test = df_test.is_attributed.values
### TRAIN RANDOMFOREST AND TUNE ON VALIDATION ###

parameters = {'n_estimators':[50,100,200]}
rfc = RandomForestClassifier(random_state=33)
model = GridSearchCV(estimator=rfc, param_grid=parameters, scoring='roc_auc',
                     cv=PredefinedSplit([-1]*len(y_train)+[0]*len(y_val)))

model.fit(np.vstack([train_feat, val_feat]), np.hstack([y_train, y_val]))

print(model.best_params_)
print(model.best_score_)
print(roc_auc_score(y_test, model.predict_proba(test_feat)[:,1]))
print(classification_report(y_test, model.predict(test_feat)))
confusion_matrix(y_test, model.predict(test_feat))
### VISUAL REPRESENTATION ###

scaler = StandardScaler()
scaler.fit(train_feat)

tsne = TSNE(n_components=2, random_state=33, n_iter=300, perplexity=3)
T = tsne.fit_transform(scaler.transform(val_feat))
plt.figure(figsize=(12,8))
plt.scatter(T[:,0], T[:,1], c=df_val.app.astype(int), cmap='plasma')<jupyter_output><empty_output><jupyter_text>Group Count + Entropy<jupyter_code>### CREATE ENCODING FOR EACH CATEGORICAL FEATURES ###

diz_enc = {}

for g_f in features:
    
    passive_feat = features[:]
    passive_feat.remove(g_f)
    
    diz_group = defaultdict(list)
    for f in passive_feat:
        
        print('--- groupby:', g_f, '###', f, '---')
        df = df_train.copy()
        df.reset_index(inplace=True)
        
        group = df.groupby([g_f, f])['index'].count()
        group = group.unstack().fillna(0)
        
        ent = group.apply(entropy, axis=1)
        for i,e in ent.to_dict().items():
            diz_group[i].extend([e])
        
        diz_group[0] = [.0]*len(passive_feat)
        
    diz_enc[g_f] = diz_group
### MAP CLASSES TO VECTORS ###

train_feat = df_train.copy()
val_feat = df_val.copy()
test_feat = df_test.copy()

for f in features:
    train_feat[f] = train_feat[f].map(diz_enc[f])
    val_feat[f] = val_feat[f].map(diz_enc[f])
    test_feat[f] = test_feat[f].map(diz_enc[f])
    
train_feat = [list(itertools.chain(*raw[1:])) for raw in train_feat[features].itertuples()]
val_feat = [list(itertools.chain(*raw[1:])) for raw in val_feat[features].itertuples()]
test_feat = [list(itertools.chain(*raw[1:])) for raw in test_feat[features].itertuples()]
### TRAIN RANDOMFOREST AND TUNE ON VALIDATION ###

parameters = {'n_estimators':[50,100,200]}
rfc = RandomForestClassifier(random_state=33)
model = GridSearchCV(estimator=rfc, param_grid=parameters, scoring='roc_auc',
                     cv=PredefinedSplit([-1]*len(y_train)+[0]*len(y_val)))

model.fit(np.vstack([train_feat, val_feat]), np.hstack([y_train, y_val]))

print(model.best_params_)
print(model.best_score_)
print(roc_auc_score(y_test, model.predict_proba(test_feat)[:,1]))
print(classification_report(y_test, model.predict(test_feat)))
confusion_matrix(y_test, model.predict(test_feat))
### VISUAL REPRESENTATION ###

scaler = StandardScaler()
scaler.fit(train_feat)

tsne = TSNE(n_components=2, random_state=33, n_iter=300, perplexity=3)
T = tsne.fit_transform(scaler.transform(val_feat))
plt.figure(figsize=(12,8))
plt.scatter(T[:,0], T[:,1], c=df_val.app.astype(int), cmap='plasma')<jupyter_output><empty_output><jupyter_text>Group2Vec<jupyter_code>### DEFINE GROUP2VEC MODEL ###

def set_seed(seed):
    
    tf.random.set_seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    random.seed(seed)

def get_embed(inp, size, emb_dim, name):

    emb = Embedding(size, emb_dim)(inp)
    emb = Flatten(name=name)(emb)
    
    return emb

def Group2Vec(df, group_key, emb_dim=8):
    
    set_seed(33)
    
    opt = Adam(lr=0.0001, clipnorm=0.5)
                
    inps = []
    embs = {}
    for g_k in group_key:
        inp = Input((1,), name=g_k+'_inp')
        embs[g_k] = get_embed(inp, int(df[g_k].max())+1, emb_dim, g_k)
        inps.append(inp)
    
    outs = []
    for g_k in group_key:
        
        new_group_key = group_key[:]
        new_group_key.remove(g_k)
        
        dots = [Dot(axes=1)([embs[k], embs[g_k]]) for k in new_group_key]
        dot_product = Average()(dots)
        outs.append(dot_product)
        
    x = Concatenate()(outs)
    x = Dropout(0.5)(x)
    out = Dense(1, activation='sigmoid')(x)
    
    model = Model(inputs=inps, outputs=out)
    model.compile(optimizer=opt, loss='binary_crossentropy')
    
    return model
model = Group2Vec(df_train, group_key=features, emb_dim=16)
model.summary()
### FIT GROUP2VEC ###

model = Group2Vec(df_train, group_key=features, emb_dim=16)

es = EarlyStopping(patience=5, verbose=1, min_delta=0.001, monitor='val_loss', mode='auto', restore_best_weights=True)
history = model.fit([df_train[f].values for f in features], y_train, 
                     epochs=100, batch_size=512, verbose=2, callbacks=[es],
                     validation_data=([df_val[f].values for f in features], y_val))
pred_val = model.predict([df_val[f].values for f in features])
print(pred_val.shape)

pred_test = model.predict([df_test[f].values for f in features])
print(pred_test.shape)
print(roc_auc_score(y_test, pred_test))
print(classification_report(y_test, (pred_test>0.5)+0))
confusion_matrix(y_test, (pred_test>0.5)+0)
### DEFINE EMBEDDING EXTRACTOR ###

emb_model = Model([model.get_layer(f+'_inp').input for f in features], 
                  [model.get_layer(f).output for f in features])
### EXTRACT EMBEDDINGS ###

emb_train = emb_model.predict([df_train[f].values for f in features])
emb_train = np.hstack(emb_train)
print(emb_train.shape)

emb_val = emb_model.predict([df_val[f].values for f in features])
emb_val = np.hstack(emb_val)
print(emb_val.shape)

emb_test = emb_model.predict([df_test[f].values for f in features])
emb_test = np.hstack(emb_test)
print(emb_test.shape)
### VISUAL REPRESENTATION ###

tsne = TSNE(n_components=2, random_state=33, n_iter=300, perplexity=3)
T = tsne.fit_transform(emb_val)
plt.figure(figsize=(12,8))
plt.scatter(T[:,0], T[:,1], c=df_val.app.astype(int), cmap='plasma')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15743172268907563, 'fraction_numerical': 0.01641281512605042, 'mean_word_length': 4.30062630480167, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '319604', 'license': 'MIT', 'n_tokens_mistral': 2997, 'n_tokens_neox': 2952, 'n_words': 463, 'path': 'Group2Vec/Group2Vec.ipynb', 'repo_name': 'MugiPham/MEDIUM_NoteBook', 'size': '7616'}