<jupyter_start><jupyter_text>Issue with the varying FDP curves<jupyter_code>import numpy as np
import numpy.matlib as npm
import sys
import matplotlib.pyplot as plt 

import sanssouci as sa

import random
random.seed(a=2022)

m = 2000         # number of variables     (eg : number of voxels in an image)
n = 80            # number of observations  (eg : number of images)
pi0 = 0.99        # proportion of noise (true null hypotheses)
p0 = int(np.round(pi0*m)) # number of true null hypotheses (non active voxels)
rho = 0.3        # equi-correlation
s = 10             # signal to noise ratio (because sigma is 1 CHECK!)

# generate noise
Z=np.random.randn(n,m)
w=np.random.randn(n)
W=Z.copy()
for i in range(m):
  W[:,i]=w[:] # W has the same w for each subject to ensure the dependence
Y=np.sqrt(1-rho)*Z + np.sqrt(rho)*W

# cov(Y_n(v), Y_m(v)) = E( sqrt(1-rho)z_n + sqrt(rho)*w,sqrt(1-rho)z_m + sqrt(rho)*w) = (1-rho)delta_{nm} + rho
# (as var(w) = 1 = var(z_n) = var(z_m)) (so the text above is wrong!)

# add signal (for false null hypotheses)
categ=np.random.binomial(1, 0.5, size=n) # Equal probability of each - but ususally not balanced in fact!
data = Y.copy()
w1=np.where(categ==1)<jupyter_output><empty_output><jupyter_text>Can toggle this to see the effect<jupyter_code># Add signal to the data when the category is 1! :) 
data[w1, (p0+1):m] = Y[w1, (p0+1):m] - s
rwt = sa.row_welch_tests(data, categ)
pvals_perm = rwt['p_value']
plt.hist(pvals_perm)
lambda_set = np.arange(10)/10
print(lambda_set)
for I in np.arange(len(lambda_set)):
    thresh = sa.t_linear(lambda_set[I], np.arange(1,m+1), m)
    max_FDP_perm = sa.curve_max_fp(pvals_perm, thresh)
    plt.plot(max_FDP_perm)<jupyter_output>[0.  0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9]
The input p-values were not sorted -> this is done now
The input p-values were not sorted -> this is done now
The input p-values were not sorted -> this is done now
The input p-values were not sorted -> this is done now
The input p-values were not sorted -> this is done now
The input p-values were not sorted -> this is done now
The input p-values were not sorted -> this is done now
The input p-values were not sorted -> this is done now
The input p-values were not sorted -> this is done now
The input p-values were not sorted -> this is done now
{'fraction_non_alphanumeric': 0.12048716833405829, 'fraction_numerical': 0.0274032187907786, 'mean_word_length': 3.7817047817047817, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '230680', 'license': 'MIT', 'n_tokens_mistral': 889, 'n_tokens_neox': 843, 'n_words': 295, 'path': 'examples/Potential Bugs/.ipynb_checkpoints/Issue with FDP curves-checkpoint.ipynb', 'repo_name': 'alexblnn/pyrft', 'size': '2299'}