<jupyter_start><jupyter_text>Get histone mod. ChIP-seq datasets from Roadmap EpigenomicsWe're limiting our training on histone modification ChIP-seq datasets targeting the following modifications:- H3K4me1- H3K4me3- H3K27ac- H3K9ac- H3K27me3- H3K9me3- H3K36me3The list of all datasets is available at https://egg2.wustl.edu/roadmap/data/byFileType/signal/consolidated/macs2signal/foldChange/<jupyter_code>%load_ext autoreload
%autoreload 2
%matplotlib inline<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>Imports<jupyter_code>import io
import json
import numpy as np
import os
import pandas as pd
import requests
import sys
from urllib.parse import urlencode, urljoin

module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path:
    sys.path.append(module_path)
module_path = os.path.abspath(os.path.join('../experiments'))
if module_path not in sys.path:
    sys.path.append(module_path)
base = '..'
datasets_file = 'roadmap-epigenomics.tsv'
mods = ['H3K4me1', 'H3K4me3', 'H3K27ac', 'H3K27me3', 'H3K9ac', 'H3K9me3', 'H3K36me3']
roadmap_datasets = pd.read_csv(os.path.join(base, 'data', datasets_file), sep='\t', header=None, names=('id', 'target', 'size'))
from functools import reduce

filters = [roadmap_datasets['filename'].str.lower().str.contains(mod.lower()) for mod in mods]
all_filters = reduce(lambda a, b: a | b, filters)
selected_datasets = roadmap_datasets[all_filters]
"""Select experiments that feature all targets"""
tmp = selected_datasets.groupby(['id']).agg(['count'])
tmp = tmp['target'] == 7
tmp = tmp.index[tmp['count'].values]
datasets_with_all_targets = selected_datasets[selected_datasets['id'].isin(np.array(tmp))]
e_ids = datasets_with_all_targets['id'].unique().tolist()<jupyter_output><empty_output><jupyter_text>Estimate size<jupyter_code>N = len(roadmap_datasets)
subsample = [72, 72, 72, 54, 54, 54, 54]
selected = None

for i, f in enumerate(filters):
    if selected is None:
        selected = np.random.choice(np.arange(N)[f], subsample[i], replace=False)
    else:
        selected = np.concatenate((selected, np.random.choice(np.arange(N)[f], subsample[i], replace=False)))
        
roadmap_datasets.iloc[selected]['size'].sum() / 1000<jupyter_output><empty_output><jupyter_text>Save JSON<jupyter_code>with open('../datasets-chip-histone-mod-hg19.json'.format(), 'w') as f:
    json.dump(e_ids, f)<jupyter_output><empty_output><jupyter_text>Download data<jupyter_code>from download import download_roadmap_epigenomics

with open('../settings-roadmap-epigenomics-chip-histone-mod-12kb-testing.json'.format(), 'r') as f:
    settings = json.load(f)

download_roadmap_epigenomics(
    [e_ids[0]],
    settings,
    base='..'
)
from bedtobigbed import bed_to_bigbed

bed_to_bigbed(
    [e_ids[0]],
    settings,
    base='..',
    clear=True,
)
from prepare import get_roadmap_datasets, prepare

prepare(
    'chip',
    get_roadmap_datasets([e_ids[0]], 'chip'),
    settings,
    roadmap=True,
    clear=True,
    verbose=True,
    silent=False,
)
from evaluate import evaluate
loss = evaluate(
    model_name = 'cf-32-64-128--ck-3-5-7--du-1024-256--do-0-0-0-0-0--e-10--rl-0--o-adadelta--lr-1.0--lrd-0.001--l-smse-10--m---bn-0-0-0-0-0-0--bni-0--pw-8--sw-logn--swz-0',
    dataset_name = 'merged',
    base = '..',
    clear = True,
    verbose = True,
)
loss<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15197657393850658, 'fraction_numerical': 0.03367496339677892, 'mean_word_length': 4.951219512195122, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '344387', 'license': 'Apache-2.0', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1277, 'n_words': 239, 'path': 'experiments/notebooks/Get histone mod ChIP-seq dataset file accessions from Roadmap Epigenomics.ipynb', 'repo_name': 'flekschas/peax', 'size': '3415'}