<commit_before># Renames the file by replacing the extract ID with the sample code
# path relative or absolute pathname
# pattern pattern to identify files that should be renamed
# mapping the mapping of extract ID (names of the vector) to sample code (entries in the vector)
# start position of the first character of the extract ID in the file name
# end position after the last character of the extract ID in the file name
replaceExtractId = function(path, pattern, mapping, start, end) {
	if (!require(stringr)) {
		stop("Could not load required package \"stringr\".")
  	}
  
  lapply(list.files(path=path, pattern=pattern, full.names=TRUE),
  			 function(x, mapping, start, end) {
  			 	 extract = str_sub(x, start=start, end=end)
  			 	 file.rename(from=x, 
  			 	 			 to=str_replace(x,
  			 	 			 pattern=extract, 
  			 	 			 replacement=mapping[extract, 1]))
  			 },
  			 mapping=mapping,
  			 start=start,
  			 end=end)
}

# Read raw TCGA data
# path absolute or relative directory name
# pattern file name pattern of the files to read in 
# what vector giving the data types of the columns to read, will be passed on to "scan"
# sampleType either tumor or normal; if "tumor", only samples with tissue types 1 through 9 will
# be read; if "normal", only samples with tissue types 10 and 11 will be read
# Returns a list with one entry per file. The sample barcode is used as element name
readRawData = function(path, pattern, what, sampleType=c("tumor", "normal")) {
	if (!require(stringr)) {
		stop("Could not load required package \"stringr\".")
	}
	
	sampleType = match.arg(sampleType)
	
	# Assume tumor by default
	typeCodes = 1:9
	if (sampleType == "normal") {
		typeCodes = 10:11
	}
	
	read = list()
	for (f in list.files(path=path, pattern=pattern, full.names=TRUE)) {
		# Get the start of the sample barcode
		loc = str_locate(f, "TCGA")[1, 1]
		# the full barcode
		barcode = str_sub(f, start=loc, end=loc+11)
		# and the sample type
		sampleType = str_sub(f, start=loc+13, end=loc+14)
		if (as.integer(sampleType) %in% typeCodes) {
			read[[barcode]] = scan(f, 
					what=what, 
					skip=1, 
					flush=TRUE, 
					strip.white=TRUE, 
					fill=TRUE, 
					sep="\t",
					quiet=TRUE)
		}
	}
	
	read
}

##' Read raw TCGA data<commit_msg>Adds skip parameter for readRawData<commit_after># Renames the file by replacing the extract ID with the sample code
# path relative or absolute pathname
# pattern pattern to identify files that should be renamed
# mapping the mapping of extract ID (names of the vector) to sample code (entries in the vector)
# start position of the first character of the extract ID in the file name
# end position after the last character of the extract ID in the file name
replaceExtractId = function(path, pattern, mapping, start, end) {
	if (!require(stringr)) {
		stop("Could not load required package \"stringr\".")
  	}
  
  lapply(list.files(path=path, pattern=pattern, full.names=TRUE),
  			 function(x, mapping, start, end) {
  			 	 extract = str_sub(x, start=start, end=end)
  			 	 file.rename(from=x, 
  			 	 			 to=str_replace(x,
  			 	 			 pattern=extract, 
  			 	 			 replacement=mapping[extract, 1]))
  			 },
  			 mapping=mapping,
  			 start=start,
  			 end=end)
}

# Read raw TCGA data
# path absolute or relative directory name
# pattern file name pattern of the files to read in 
# what vector giving the data types of the columns to read, will be passed on to "scan"
# sampleType either tumor or normal; if "tumor", only samples with tissue types 1 through 9 will
# be read; if "normal", only samples with tissue types 10 and 11 will be read
# skip number of lines to skip; default: 1
# Returns a list with one entry per file. The sample barcode is used as element name
readRawData = function(path, pattern, what, sampleType=c("tumor", "normal"), skip=1) {
	if (!require(stringr)) {
		stop("Could not load required package \"stringr\".")
	}
	
	sampleType = match.arg(sampleType)
	
	# Assume tumor by default
	typeCodes = 1:9
	if (sampleType == "normal") {
		typeCodes = 10:11
	}
	
	read = list()
	for (f in list.files(path=path, pattern=pattern, full.names=TRUE)) {
		# Get the start of the sample barcode
		loc = str_locate(f, "TCGA")[1, 1]
		# the full barcode
		barcode = str_sub(f, start=loc, end=loc+11)
		# and the sample type
		sampleType = str_sub(f, start=loc+13, end=loc+14)
		if (as.integer(sampleType) %in% typeCodes) {
			read[[barcode]] = scan(f, 
					what=what, 
					skip=skip, 
					flush=TRUE, 
					strip.white=TRUE, 
					fill=TRUE, 
					sep="\t",
					quiet=TRUE)
		}
	}
	
	read
}

##' Read raw TCGA data
{'fraction_non_alphanumeric': 0.10308159722222222, 'fraction_numerical': 0.009765625, 'mean_word_length': 3.4617618586640853, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1825717', 'n_tokens_mistral': 1661, 'n_tokens_neox': 1455, 'n_words': 548}