<jupyter_start><jupyter_text>Explore Error Termswhy does the validation set outperform the traning set?<jupyter_code>import logging
from   setupLogging import setupLogging
configFilePath = setupLogging( default_path='src/test/logging.test.ini.json')
logger = logging.getLogger("notebook")
logger.info("using logging configuration file:{}".format(configFilePath))

from   DEMETER2.lowRankMatrixFactorizationEasyOfUse \
            import LowRankMatrixFactorizationEasyOfUse as LrmfEoU

import numpy as np
dataDir = "data/"
dataFileName = "D2_Achilles_gene_dep_scores.tsv"
numFeatures = 19
geneFilterPercent = 0.25 
holdOutPercent = 0.40 
easyOfUse = LrmfEoU(dataDir, dataFileName, numFeatures, geneFilterPercent, holdOutPercent)
resultsDict = easyOfUse.loadAll()
# clean tidy version of demeter data
Y, R, cellLinesNames, geneNames, = resultsDict["DEMETER2"]
geneDependencies = Y
print("geneDependencies.shape:{}".format(geneDependencies.shape))

# trained model
# scipy.optimize.OptimizeResult
X, Theta, optimizeResult = resultsDict["LowRankMatrixFactorizationModel"]
genes = X
print("genes.shape:{}".format(genes.shape))
cellLines = Theta
print("cellLines.shape:{}".format(cellLines.shape))

# knockout logical filters. Use to select Y Train, Validations, and Test values
RTrain, RValidation, RTest = resultsDict["filters"]

print("number of observed values in traning set: {}".format(np.sum(RTrain)))
print ("size of traning set: {}".format(np.size(RTrain)))<jupyter_output>geneDependencies.shape:(11193, 501)
genes.shape:(11193, 19)
cellLines.shape:(501, 19)
number of observed values in traning set: 3364160
size of traning set: 5607693<jupyter_text>Select the training set and explore<jupyter_code>train = np.multiply(Y, RTrain)
print ("size of traning set: {}".format(np.size(train)))
print("train set descriptive stats\n")
print( "min  : {:+6.4f}".format(np.min(train) ))
print( "max  : {:+6.4f}".format(np.max(train) ))
print( "mean : {:+6.4f}".format(np.mean(train) ))
print( "std  : {:+6.4f}".format(np.std(train) ))
print( "0.25 : {:+6.4f}".format(np.percentile(train, 0.25) ))
print( "0.50 : {:+6.4f}".format(np.percentile(train, 0.50) ))
print( "0.75 : {:+6.4f}".format(np.percentile(train, 0.75) ))<jupyter_output>train set descriptive stats

min  : -2.6574
max  : +1.7028
mean : -0.0360
std  : +0.1915
0.25 : -1.2744
0.50 : -1.1213
0.75 : -1.0035
{'fraction_non_alphanumeric': 0.14866581956797967, 'fraction_numerical': 0.051249470563320625, 'mean_word_length': 5.988165680473373, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '381728', 'license': 'MIT', 'n_tokens_mistral': 930, 'n_tokens_neox': 831, 'n_words': 187, 'path': 'exploreErrorTerms.ipynb', 'repo_name': 'aedavids/lab3RotationProject', 'size': '2361'}