<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/python

"""
This file contains experimental results related to the PCA / 
lower-dimensional subspace conception of semantic complexity.

52% of the training data is less than 50 words, so this would
  be a very unrepresentative sample
19% of the training data is less than 30 words
8% of the training data is less than 20 words
0.6% of the training data is less than 5 words

Usage:
From top level (/path/to/repo/icgauge), issue the command 
     `python -m experiments.pca`
"""


import json
from collections import defaultdict
from matplotlib import pyplot as plt

import scipy
import numpy as np

import icgauge
from icgauge import experiment_frameworks
from icgauge import feature_extractors

run_experiment = True

if run_experiment:
  corr, alpha, conf_matrix, details = experiment_frameworks.experiment_features_iterated(
      train_reader=icgauge.data_readers.train_and_dev, 
      assess_reader=None, #icgauge.data_readers.test_official, 
      train_size=0.7,
      phi_list=[
        #        icgauge.feature_extractors.number_words_only,
        #        icgauge.feature_extractors.dimensional_decomposition,
                 icgauge.feature_extractors.semcom_pca_features
               ], 
      class_func=icgauge.label_transformers.identity_class_func, #ternary_class_func
      train_func=icgauge.training_functions.fit_logistic_at,#_with_crossvalidation,
      score_func=scipy.stats.stats.pearsonr,
      verbose=False,
      iterations=1)

  # Print out the results
  print "\n-- AFTER COMPLETION --"
  print "Averaged correlation (95% CI): " 
  print np.round(np.mean(corr),2), "+/-", np.round(np.std(corr),2)
  print "All correlations:"
  print corr
  print
  print "Averaged Cronbach's alpha (95% CI): " 
  print np.round(np.mean(alpha),2), "+/-", np.round(np.std(alpha),2)
  print "All alphas:"
  print alpha
  print
  print "Confusion matrix:"
  print conf_matrix

  # Store the results to disk -- "truth"/"prediction"/"example"
  with open("results.json", "w") as fp:
      json.dump(details, fp, indent=4)


"""""""""""""""""""""
Experimental results.<commit_msg>Further cleans up PCA code<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/python

"""
This file contains experimental results related to the PCA / 
lower-dimensional subspace conception of semantic complexity.

52% of the training data is less than 50 words, so this would
  be a very unrepresentative sample
19% of the training data is less than 30 words
8% of the training data is less than 20 words
0.6% of the training data is less than 5 words

Usage:
From top level (/path/to/repo/icgauge), issue the command 
     `python -m experiments.pca`
"""


import json
from collections import defaultdict
from matplotlib import pyplot as plt

import scipy
import numpy as np

import icgauge
from icgauge import experiment_frameworks
from icgauge import feature_extractors

run_experiment = True

if run_experiment:
  corr, alpha, conf_matrix, details = experiment_frameworks.experiment_features_iterated(
      train_reader=icgauge.data_readers.train_and_dev, 
      assess_reader=None, #icgauge.data_readers.test_official, 
      train_size=0.7,
      phi_list=[
        # These two features were used to control for the effect of number
        # of words and to get at the effect of the decomposition itself
        #        icgauge.feature_extractors.number_words_only,
        #        icgauge.feature_extractors.dimensional_decomposition,
        # The real feature function for PCA, though, is:
                 icgauge.feature_extractors.semcom_pca_features
               ], 
      class_func=icgauge.label_transformers.identity_class_func, #ternary_class_func
      train_func=icgauge.training_functions.fit_logistic_at,#_with_crossvalidation,
      score_func=scipy.stats.stats.pearsonr,
      verbose=False,
      iterations=1)

  # Print out the results
  print "\n-- AFTER COMPLETION --"
  print "Averaged correlation (95% CI): " 
  print np.round(np.mean(corr),2), "+/-", np.round(np.std(corr),2)
  print "All correlations:"
  print corr
  print
  print "Averaged Cronbach's alpha (95% CI): " 
  print np.round(np.mean(alpha),2), "+/-", np.round(np.std(alpha),2)
  print "All alphas:"
  print alpha
  print
  print "Confusion matrix:"
  print conf_matrix

  # Store the results to disk -- "truth"/"prediction"/"example"
  with open("results.json", "w") as fp:
      json.dump(details, fp, indent=4)


"""""""""""""""""""""
Experimental results.
{'fraction_non_alphanumeric': 0.09868859746610358, 'fraction_numerical': 0.012002667259390976, 'mean_word_length': 3.677754677754678, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5773636', 'n_tokens_mistral': 1481, 'n_tokens_neox': 1367, 'n_words': 409}