<commit_before>import os
import argparse
from pipelines.builder import PipelineBuilder
from pipelines.schema import PipelineSchema
from pipelines.utils import PipelinesConfig, DataUtils

# Parse Arguments
parser = argparse.ArgumentParser()
parser.add_argument("--analysisId", required=True)
parser.add_argument("--cghubKey", required=False)
parser.add_argument("--cghubOutput", required=True)
parser.add_argument("--cghubScript", required=True)
parser.add_argument("--cghubLogs", required=True)
parser.add_argument("--fastqcOutput", required=True)
parser.add_argument("--fastqcLogs", required=True)
parser.add_argument("--samtoolsIndexLogs", required=True)
parser.add_argument("--setMetaLogs", required=True)
parser.add_argument("--preemptible", action="store_true", default=False, required=False)
args = parser.parse_args()

# Build "Incoming Data" Pipeline

configPath = os.path.join(os.environ["HOME"], ".isb-cgc-pipelines", "config")

pipelinesConfig = PipelinesConfig(configPath)

pipelineBuilder = PipelineBuilder(pipelinesConfig)

setMetaFiles = []

files = DataUtils.getAnalysisDetail(args.analysisId)["result_set"]["results"][0]["files"]

objectPath = DataUtils.constructObjectPath(args.analysisId, args.cghubOutput)

cghubSchema = PipelineSchema("cghub", pipelinesConfig, args.cghubLogs, "b.gcr.io/isb-cgc-public-docker-images/gtdownload",
                             scriptUrl=args.cghubScript,
                             cores=4,
                             diskSize=DataUtils.calculateDiskSize(analysisId=args.analysisId, roundToNearestGbInterval=100),
                             diskType="PERSISTENT_SSD",
                             env="ANALYSIS_ID={analysisId},CGHUB_KEY={key}".format(analysisId=args.analysisId, key=os.path.basename(args.cghubKey)),
                             outputs="{analysisId}/*:{outputPath}".format(analysisId=args.analysisId, outputPath=objectPath),
                             tag=args.analysisId,
                             preemptible=True)

pipelineBuilder.addStep(cghubSchema)

## DOWNSTREAM STEPS ##

setMetaParents = []

foundBai = False
foundBam = False
foundFastq = False


for f in files:
	ext = f["filename"].split('.')[-1]
	if ext == "bam":
		foundBam = True
		bamFileName = f["filename"]
	
	elif ext == "bai":
		foundBai = True<commit_msg>Added an input to the cghub task<commit_after>import os
import argparse
from pipelines.builder import PipelineBuilder
from pipelines.schema import PipelineSchema
from pipelines.utils import PipelinesConfig, DataUtils

# Parse Arguments
parser = argparse.ArgumentParser()
parser.add_argument("--analysisId", required=True)
parser.add_argument("--cghubKey", required=False)
parser.add_argument("--cghubOutput", required=True)
parser.add_argument("--cghubScript", required=True)
parser.add_argument("--cghubLogs", required=True)
parser.add_argument("--fastqcOutput", required=True)
parser.add_argument("--fastqcLogs", required=True)
parser.add_argument("--samtoolsIndexLogs", required=True)
parser.add_argument("--setMetaLogs", required=True)
parser.add_argument("--preemptible", action="store_true", default=False, required=False)
args = parser.parse_args()

# Build "Incoming Data" Pipeline

configPath = os.path.join(os.environ["HOME"], ".isb-cgc-pipelines", "config")

pipelinesConfig = PipelinesConfig(configPath)

pipelineBuilder = PipelineBuilder(pipelinesConfig)

setMetaFiles = []

files = DataUtils.getAnalysisDetail(args.analysisId)["result_set"]["results"][0]["files"]

objectPath = DataUtils.constructObjectPath(args.analysisId, args.cghubOutput)

cghubSchema = PipelineSchema("cghub", pipelinesConfig, args.cghubLogs, "b.gcr.io/isb-cgc-public-docker-images/gtdownload",
                             scriptUrl=args.cghubScript,
                             cores=4,
                             diskSize=DataUtils.calculateDiskSize(analysisId=args.analysisId, roundToNearestGbInterval=100),
                             diskType="PERSISTENT_SSD",
                             env="ANALYSIS_ID={analysisId},CGHUB_KEY={key}".format(analysisId=args.analysisId, key=os.path.basename(args.cghubKey)),
                             inputs="{cghubKey}:/{cghubKeyFileName}".format(cghubKey=args.cghubKey, cghubKeyFileName=os.path.basename(args.cghubKey)),
                             outputs="{analysisId}/*:{outputPath}".format(analysisId=args.analysisId, outputPath=objectPath),
                             tag=args.analysisId,
                             preemptible=True)

pipelineBuilder.addStep(cghubSchema)

## DOWNSTREAM STEPS ##

setMetaParents = []

foundBai = False
foundBam = False
foundFastq = False


for f in files:
	ext = f["filename"].split('.')[-1]
	if ext == "bam":
		foundBam = True
		bamFileName = f["filename"]
	
	elif ext == "bai":
		foundBai = True
{'fraction_non_alphanumeric': 0.1288163948138854, 'fraction_numerical': 0.002509410288582183, 'mean_word_length': 3.961618257261411, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5603494', 'n_tokens_mistral': 1510, 'n_tokens_neox': 1476, 'n_words': 221}