<jupyter_start><jupyter_text>Process Samples  Setup<jupyter_code># Location of docker files
common_docker_registry = "quay.io/hisplan"

prefix = "ArchRCR" # Workflow to run; also .wdl filename prefix
pipeline_type = prefix # field in *.labels.json
output_dirname = "archr-cr-results"

# If need to add comment, put here
comment = ""
# Locations of workflow-related directories and files
path_to_cromwell_secrets = f"{Path.home()}/.cromwell/cromwell-secrets.json" # CHANGE THIS
workflow_dir = f"{Path.home()}/scing/bin/wdl-ArchR" # CHANGE THIS
execp = "submit-fastq.sh" # CHANGE THIS FOR SHARP
config_dir = f"{workflow_dir}/configs"
path_to_options = f"{workflow_dir}/ArchR.options.aws.json"

# Other file locations
db_credentials_path = f"{Path.home()}/.config.json" # CHANGE THIS
# Set credentials based on SCRIdb CLI config file
with open(db_credentials_path) as f:
    creds = json.load(f)
# Samples on which to run CellRangerATAC
# Note: Assumes data is transferred to AWS S3 (this should be an s3 location)
# Note: Assumes directory name is name of sample
common_dir = "s3://dp-lab-data/collaborators/lowe/p53_loss_enrichment"
samples = [
    #"JR-1497_p489c_shRen_caer_48h_multiome_1",
    #"JR-1497_p489c_shRen_caer_48h_multiome_2",
    "JR-1497_p489c_shRen_caer_48h_multiome_3",
    #"JR-1497_p489c_shRen_caer_48h_multiome_4",
]
sample_paths = [
    f"{common_dir}/{sample}" for sample in samples
]<jupyter_output><empty_output><jupyter_text>Execution<jupyter_code># Get information for all samples
sample_paths = [s.strip('/') for s in sample_paths] # remove trailing slash if exists
sample_names = [os.path.basename(s) for s in sample_paths]
#sample_names = [s.replace("Redo_", "") for s in sample_names]
print(sample_names)
samples = pd.DataFrame(
    sample_paths,
    index=sample_names,
    columns=["S3_Path"],
    dtype=str,
)
samples["Sample_ID"] = pd.Series(samples.index).apply(
    lambda x: get_sample_id(x, creds['user'], creds['password'])
).values

# Get FASTQ paths from S3
# Note: Uses same FASTQ file ids for all samples
#fastq_file_ids = fastq_map[prefix]
samples["FASTQs"] = samples["S3_Path"].apply(
    lambda x: get_fastqs(x + "_ATAC", folder="FASTQ")
)

# Get reference genome location
samples["CR_Reference"] = samples["Sample_ID"].apply(
    lambda x: get_reference(x, "CellRanger", prefix, creds["user"], creds["password"])
)
samples["ArchR_Reference"] = samples["Sample_ID"].apply(
    lambda x: get_reference(x, "ArchR", prefix, creds["user"], creds["password"])
)
samples
# Load minimum inputs and labels fields from templates
with open(f"{config_dir}/template.fastq.inputs.json") as f:
    std_inputs_fields = list(json.load(f).keys())
    
with open(f"{config_dir}/template.fastq.labels.json") as f:
    std_labels_fields = list(json.load(f).keys())
    
# Annotate all samples with workflow inputs and labels
inputs = pd.DataFrame(index=samples.index, columns=std_inputs_fields,)
labels = pd.DataFrame(index=samples.index, columns=std_labels_fields,)

# Annotate inputs
inputs[f"{prefix}.sampleName"] = inputs.index # may need to change
inputs[f"{prefix}.fastqFiles"] = samples["FASTQs"].apply(lambda x: np.ravel(list(x.values())))
inputs[f"{prefix}.fastqNames"] = inputs[f"{prefix}.fastqFiles"].apply(lambda x: get_fastqs_name(x))
inputs[f"{prefix}.genomeCellRanger"] = samples["CR_Reference"].apply(
    lambda x: {
        "name": re.match(r'.*/refdata-cellranger-[a-z]+-(.*).tar.gz$', x)[1],
        "location": x,
    }
)
inputs[f"{prefix}.genomeArchR"] = samples["ArchR_Reference"]
inputs[f"{prefix}.numCores"] = 1
inputs[f"{prefix}.dockerRegistry"] = common_docker_registry

# Annotate labels
labels["pipelineType"] = "CellRangerARC"
labels["project"] = samples["Sample_ID"].apply(lambda x: get_project_id(x, creds["user"], creds["password"]))
labels["sample"] = labels.index
labels["owner"] = creds["user"]
labels["destination"] = samples['S3_Path'] + "/" + output_dirname
labels["transfer"] = "-"
labels["comment"] = creds["user"]

assert (std_inputs_fields == list(inputs.columns)) & (inputs.notna().values.all())
assert (std_labels_fields == list(labels.columns)) & (labels.notna().values.all())
inputs
labels
stdouts = [] # to store all outputs
process = True

with tqdm(samples.index) as t:

    for sample_name in t:

        # Write inputs and labels to file
        path_to_inputs = f"{config_dir}/{sample_name}.inputs.json"
        with open(path_to_inputs, "w") as f_inputs:
            json.dump(inputs.loc[sample_name].to_dict(), f_inputs, indent=4, cls=NpEncoder)

        path_to_labels = f"{config_dir}/{sample_name}.labels.json"
        with open(path_to_labels, "w") as f_labels:
            json.dump(labels.loc[sample_name].to_dict(), f_labels, indent=4, cls=NpEncoder)

        if process:
            stdouts.append(run(
                workflow_path = workflow_dir,
                execp = execp,
                secrets = path_to_cromwell_secrets,
                inputs = path_to_inputs,
                labels = path_to_labels,
                options = path_to_options,
            ))
stdouts<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14201647704982345, 'fraction_numerical': 0.01039623381718321, 'mean_word_length': 4.716367713004484, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '99042', 'license': 'MIT', 'n_tokens_mistral': 1821, 'n_tokens_neox': 1758, 'n_words': 363, 'path': 'notebooks/Run_ArchR.ipynb', 'repo_name': 'andrewmoorman/scing-notebooks', 'size': '5098'}