<jupyter_start><jupyter_text>Choose which files to work with<jupyter_code>version = "0"     #already completed i = 0, 1, 2, 3, 4, 5 (available: 0,1,2,3,4,5,6)

phen = '20160'
desc = 'smoking'<jupyter_output><empty_output><jupyter_text>Convert files<jupyter_code>for i in ["A","B"]:
    filename = phen+'_vds'+ i + '_s' + version
    dlpath_cloud = 'gs://nbaya/split/'
    path_local = '/Users/nbaya/Documents/lab/ukbb-sexdiff/split/'
    subprocess.call(['gsutil','cp',dlpath_cloud + filename + '.tsv.bgz', path_local])

    with gzip.open(path_local + filename + '.tsv.bgz', 'r') as f:
        f.readline()
        rows1 = []
        rows2 = []
        for line in f:
            v, va = line.strip().split(b'\t')
            json_line2 = json.loads(va)
            rows1.append(v.decode('utf-8'))
            rows2.append(json_line2)

    rsid = list(map(lambda x: x['rsid'], rows2))
    A1 = list(map(lambda x: rows1[x].split(':')[3], range(len(rows1))))
    A2 = list(map(lambda x: rows1[x].split(':')[2], range(len(rows1))))

    df_linreg = pd.DataFrame(list(map(lambda x: (x['linreg']), rows2)))
    N = df_linreg['nCompleteSamples']
    Z = list(map(lambda x: x[0], df_linreg['tstat']))

    df = pd.DataFrame({'SNP': rsid, 'A1': A1, 'A2': A2, 'N':N, 'Z': Z})

    file_path_local = path_local + filename + '.tsv.gz'
    df.to_csv(file_path_local, sep='\t', compression='gzip', index=False)
    path_cloud = 'gs://nbaya/split/test/'+filename+'.tsv.gz'
    subprocess.call(['gsutil','cp',file_path_local, path_cloud])<jupyter_output>['gsutil', 'cp', 'gs://nbaya/split/20160_vdsA_s0.tsv.bgz', '/Users/nbaya/Documents/lab/ukbb-sexdiff/split/']
['gsutil', 'cp', 'gs://nbaya/split/20160_vdsB_s0.tsv.bgz', '/Users/nbaya/Documents/lab/ukbb-sexdiff/split/']<jupyter_text>Update `{desc}.h2part.tsv`<jupyter_code>h2part_df = pd.read_csv(path_local+desc+'.h2part.tsv',delimiter='\t')
h2part_df.at[0,'female_file'] = phen+'_vdsA_s'+version+'.tsv.gz'
h2part_df.at[0,'male_file'] = phen+'_vdsB_s'+version+'.tsv.gz'
h2part_df.to_csv(path_local+desc+'.h2part.tsv',sep='\t')
subprocess.call(['gsutil','cp',path_local+desc+'.h2part.tsv','gs://nbaya/rg_sex/'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21156493804497475, 'fraction_numerical': 0.0289123451124369, 'mean_word_length': 4.382716049382716, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '162558', 'license': 'MIT', 'n_tokens_mistral': 944, 'n_tokens_neox': 910, 'n_words': 124, 'path': 'python/convert_vds.ipynb', 'repo_name': 'nikbaya/split', 'size': '2179'}