<jupyter_start><jupyter_text>1. Load combined features from SubFind & Sownak<jupyter_code># arxiv:1905.08799 talks about this stellar mass cut on page 4 top right column
output_file = 'merged_dataframe.h5'                                          
data_path = '/cosma6/data/dp004/dc-cues1/tng_dataframes/'

train, test = get_data(data_path + output_file, label)
print(len(train.columns.values))
train.sample(n=3, random_state=1)
## Balance training set in the transition region                             
center_transition, end_transition = find_transition_regions(train)
train = balance_dataset(                                                     
    train, center_transition, end_transition, sampling
)

train_features = train.drop(columns="labels")                                
train_labels = train["labels"]

test_features = test.drop(columns="labels")                                
test_labels = test["labels"]

feature_names = train_features.columns.values
## Standarize features                                                       
scaler = StandardScaler()
scaler.fit(train_features.values)
train_features_std = scaler.transform(train_features.values)
test_features_std = scaler.transform(test_features.values)

train_labels = train_labels.values
test_labels = test_labels.values

# !!!
# notice that features and labels are np.ndarrays now
# !!!<jupyter_output><empty_output><jupyter_text>2 Feature Importance 2.0 LDA decomposition<jupyter_code>def lda_transform(
    train_features, train_labels,
    test_features, test_labels,
    arg_lda):
    """
    """
    
    if isinstance(arg_lda, (float)):
        return _lda_varlimit(
            train_features, train_labels,
            test_features, test_labels,
            arg_lda,
        )
    elif isinstance(arg_lda, (dict)):
        return _lda_dict(
            train_features, train_labels,
            test_features, test_labels,
            arg_lda,
        )


def _lda_dict(
    train_features, train_labels,
    test_features, test_labels,
    arg_lda: dict):
    """
    """
    # Create LDA
    lda = LDA(**arg_lda)
    
    # tranform features based on training dataset
    train_features = lda.fit_transform(train_features, train_labels)
    test_features = lda.transform(test_features)
    
    print("explained_variance_ratio_ ", lda.explained_variance_ratio_)
        
    return train_features, test_features, lda.n_components


def _lda_varlimit(
    train_features, train_labels,
    test_features, test_labels,
    variance_limit: float):
    """
    """
    # Create array of explained variance ratios
    lda = LDA(n_components=None)
    print(train_features.shape)
    dump = lda.fit(train_features, train_labels)
    lda_var_ratios = lda.explained_variance_ratio_
    print("explained_variance_ratio_", lda_var_ratios)
    
    # Set initial variance
    total_variance = 0.0
    # Set initial number of features
    n_components = 0
    
    # Run through explained variance of each feature:
    for explained_variance in lda_var_ratios:
        
        # Add the explained variance to the total
        total_variance += explained_variance
        
        # Add one to the number of components
        n_components += 1
        
        if total_variance >= variance_limit:
            break
    
    # Create LDA
    lda = LDA(n_components=n_components, priors=None)
    
    # tranform features based on training dataset
    train_features = lda.fit_transform(train_features, train_labels)
    test_features = lda.transform(test_features)
    
    # check mean accuracy
    #score_lda = lda.score(test_features, test_labels)
        
    return train_features, test_features, lda.n_components


print("Original feature space dimensions = %d \n" %
      train_features_std.shape[1])
#"""
arg_lda = 0.95
train_features_manual, test_features_manual, n_components_manual = lda_transform(
    train_features_std, train_labels,
    test_features_std, test_labels,
    arg_lda,
)
#"""
print("n_components_manual:", n_components_manual, train_features_manual.shape)
print(" ")

arg_lda = {'solver': 'eigen', 'shrinkage': 'auto'}
train_features_dict, test_features_dict, n_components_dict = lda_transform(
    train_features_std, train_labels,
    test_features_std, test_labels,
    arg_lda,
)

print("n_components_dict:", n_components_dict, train_features_dict.shape)
plt.figure()

plt.scatter(
    train_features_dict[np.where(train_labels)[0], 0],
    [0]*len(np.where(train_labels)[0]),
    alpha=.8, label='Luminos',
)
plt.scatter(
    train_features_dict[np.where(~train_labels)[0], 0],
    [1]*len(np.where(~train_labels)[0]),
    alpha=.8, label='Dark',
)

plt.legend(loc='best', shadow=False, scatterpoints=1)
plt.show()<jupyter_output>/cosma/home/dp004/dc-beck3/.local/lib/python3.6/site-packages/matplotlib/font_manager.py:1331: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))<jupyter_text>2.1 Pearson correlation score Before LDA<jupyter_code>visualizer = Rank2D(features=feature_names, algorithm='pearson')

visualizer.fit(train_features_std, test_labels.values)
visualizer.transform(train_features_std)  
visualizer.poof()<jupyter_output><empty_output><jupyter_text>After LDA - manual<jupyter_code>visualizer = Rank2D(
    features=['PCA_%d' % dd for dd in range(train_features_pca_manual.shape[1])],
    algorithm='pearson',
)

visualizer.fit(train_features_pca_manual, train_labels.values)
visualizer.transform(train_features_pca_manual)  
visualizer.poof()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09617447264926707, 'fraction_numerical': 0.010010725777618877, 'mean_word_length': 3.504830917874396, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '552074', 'license': 'MIT', 'n_tokens_mistral': 1834, 'n_tokens_neox': 1768, 'n_words': 330, 'path': 'notebooks/LDA.ipynb', 'repo_name': 'florpi/GaHaCo', 'size': '5594'}