<commit_before>"""
Examines the principal components of the data set.
"""

from pml.api import *

import util

def examine_principal_components(dataset):
    print "Examining principal components for the data set."
    print "Percent variance per principal component:"
    print get_pct_variance_per_principal_component(dataset)
    plot_pct_variance_per_principal_component(dataset)
    
    print "Recommended number of principal components for 95% variance:"
    print recommend_num_components(dataset, min_pct_variance=0.95)

def main():
    # The original data set.
    data = util.load_data()
    
    # Fill in missing values with the average for that course.
    data.fill_missing_with_feature_means()
    
    # Count successful and probation students as one group (s)
    # Comment this out to try and distinguish all 3 groups (s, p, f)
    data.combine_labels(["s", "p"], "s")
    
    examine_principal_components(data)

if __name__ == "__main__":
    main()
    <commit_msg>Added method to pca exploration which looks up the features with the most impact.<commit_after>"""
Examines the principal components of the data set.
"""

from pml.api import *

import util

def examine_principal_components(dataset):
    print "Examining principal components for the data set."
    print "Percent variance per principal component:"
    print get_pct_variance_per_principal_component(dataset)
    plot_pct_variance_per_principal_component(dataset)
    
    print "Recommended number of principal components for 95% variance:"
    print recommend_num_components(dataset, min_pct_variance=0.95)

def pca_find_important_features(dataset):
    # The weight matrix that is used to transform the original data 
    # to the reduced data can be used to see which features are most 
    # important.  The features with the largest magnitude weight have 
    # the largest impact on the reduced data.
    
    util.print_line_break()
    print "First principal component impacts (absolute value of weight):"
    print pca(dataset, 2).get_first_component_impacts()

def main():
    # The original data set.
    data = util.load_data()
    
    # Fill in missing values with the average for that course.
    data.fill_missing_with_feature_means()
    
    # Count successful and probation students as one group (s)
    # Comment this out to try and distinguish all 3 groups (s, p, f)
    data.combine_labels(["s", "p"], "s")
    
    examine_principal_components(data)
    
    pca_find_important_features(data)

if __name__ == "__main__":
    main()
    
{'fraction_non_alphanumeric': 0.07145661562622693, 'fraction_numerical': 0.005104043973301924, 'mean_word_length': 3.844106463878327, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5267483', 'n_tokens_mistral': 739, 'n_tokens_neox': 722, 'n_words': 257}