<jupyter_start><jupyter_text>Prepare metadataThis notebook consolidates the preparation of metadata for the analyses. It requires that narps.py or PrepareMaps.ipynb has already been run.<jupyter_code>import os,sys,glob,warnings
import matplotlib.pyplot as plt
import numpy,pandas
import nilearn.input_data

from narps import Narps,NarpsDirs
from utils import get_merged_metadata_decisions

# set up directories

if 'NARPS_BASEDIR' in os.environ:
    basedir = os.environ['NARPS_BASEDIR']
else:
    basedir = '/data'
assert os.path.exists(basedir)

metadata_dir = os.path.join(basedir,'metadata')
if not os.path.exists(metadata_dir):
    os.mkdir(metadata_dir)
narps = Narps(basedir,overwrite=False)
narps.load_data()

# check zstat data
missing_zstat = {}
for teamID in narps.complete_image_sets:
    for hyp in range(1,10):
        if not hyp in narps.teams[teamID].images['unthresh']['zstat']:
            if not teamID in missing_zstat:
                missing_zstat[teamID]=[]
            missing_zstat[teamID].append(hyp)
        
print('teams with missing zstat images:')
print(missing_zstat)
# get original image and decision metadata

alldata_df = get_merged_metadata_decisions(narps.metadata_file,os.path.join(narps.dirs.dirs['orig'],'narps_results.xlsx'))
print('found merged metadata for %d teams'%alldata_df.teamID.unique().shape[0])

# change type of varnum to int
alldata_df['varnum']=alldata_df['varnum'].astype('int')

# recode variables to make analysis cleaner

def package_recoder(p):
    others = ['nistats','PALM','randomise']
    if not isinstance(p,str):
        return('Other')
    if p.find('SPM')==0:
        return('SPM')
    elif p in others:
        return('Other')
    else:
        return p

alldata_df['software'] = [package_recoder(x) for x in alldata_df['TSc_SW']]


# load smoothness data and 
smoothness_df = pandas.read_csv(os.path.join(metadata_dir,'smoothness_est.csv'))

print("found smoothness data for %d teams"%len(smoothness_df.teamID.unique()))
print('missing smoothness data for:')
print(set(narps.complete_image_sets).difference(set(smoothness_df.teamID.unique())))

# 16IN: used nonstandard stats, not usable for group analysis

alldata_df = pandas.merge(alldata_df,smoothness_df,how='left',
                          left_on=['teamID','varnum'],right_on=['teamID','hyp'])

# average FWHM estimated as:
# AvgFWHM = RESELS^(1/3) (multplied by 2 since this value is in voxels rather than mm)
# per: https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=FSL;e792b5da.0803

alldata_df['fwhm'] = [i**(1/3.)*2 for i in alldata_df.resels]
# save data for loading into R
alldata_df.to_csv(os.path.join(metadata_dir,'all_metadata.csv'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12379540400296515, 'fraction_numerical': 0.00815418828762046, 'mean_word_length': 4.893013100436681, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '599193', 'license': 'MIT', 'n_tokens_mistral': 976, 'n_tokens_neox': 948, 'n_words': 205, 'path': 'analysis/original_notebooks/PrepareMetadata.ipynb', 'repo_name': 'glatard/narps', 'size': '2698'}