<commit_before>import smbl
import snakemake
import os
import shutil

__FASTAFILES = set()

def register_fasta(fastafile):
	__FASTAFILES.add(fastafile)

def get_registered_fastas():
	return list(__FASTAFILES)

def get_fa_file_path(fasta):
	return os.path.join(smbl.fa_dir,fasta)

def fasta_file(name,address):
	return FastaFile(name=name,address=address).get()


class FastaFile:
	verbosity=False

	def __init__(self,name,address):
		self.name=name
		self.filename_short=get_fa_file_path("{}.fa".format(name))
		self.filename_full=os.path.abspath(self.filename_short)
		self.address=address
		self.src_dir=os.path.join(smbl.src_dir,self.name)

		if self.address[-3:]==".gz":
			self.format="gz"
		elif self.address[-5:]==".2bit":
			self.format="2bit"
		else:
			self.format="fa"

		register_fasta(self)
		
	def get(self):
		return self.filename_full

	def get_required_programs(self):
		if self.format=="2bit":
			return [smbl.prog.TWOBITTOFA]
		else:
			return []

	def install_all_steps(self):
		self.status_message("Installation started")
		self.install_pre()
		self.install()
		self.install_post()
		self.status_message("Installation completed")

	def install_pre(self):
		try:
			shutil.rmtree(self.src_dir)
		except FileNotFoundError:
			pass
		self.shell('mkdir -p "{src_dir}"'.format(src_dir=self.src_dir))

	def install(self):
		if self.format=="gz":
			archive=self.download(
				filename_full=self.abs_from_short("{}.gz".format(self.name))
			)
			self.extract_gz(
				filename_full_archive=archive,
				filename_full_file=self.filename_full,
			)
		elif self.format=="2bit":
			archive=self.download(
				filename_full=self.abs_from_short("{}.2bit".format(self.name))
			)
			self.extract_2bit(
				filename_full_archive=archive,
				filename_full_file=self.filename_full,
			)
		elif self.format=="fa":
			self.download(
				filename_full=self.filename_full
			)
		else:
			em="Unknown format '{}'".format(self.format)
			smbl.messages.error(em,program="SMBL")
			raise NotImplementedError(em)

	def install_post(self):
		self.shell('rm -fR "{src_dir}"'.format(src_dir=self.src_dir))

	def download(self,filename_full):
		self.status_message("Downloading a file: "+self.address)
		self.shell('mkdir -p "{dir}"'.format(dir=os.path.dirname(filename_full)))
		try:
			self.shell('curl -L --insecure -o "{fn}" "{address}"'.format(
					fn=filename_full,
					address=self.address,
				))
		except:
			self.shell('curl -L --insecure -o "{fn}" "{address}"'.format(
					fn=filename_full,
					address=self.address
				))
		return filename_full

	def extract_gz(self,filename_full_archive,filename_full_file):<commit_msg>Wget as alternative method of download also for FASTA files.<commit_after>import smbl
import snakemake
import os
import shutil

__FASTAFILES = set()

def register_fasta(fastafile):
	__FASTAFILES.add(fastafile)

def get_registered_fastas():
	return list(__FASTAFILES)

def get_fa_file_path(fasta):
	return os.path.join(smbl.fa_dir,fasta)

def fasta_file(name,address):
	return FastaFile(name=name,address=address).get()


class FastaFile:
	verbosity=False

	def __init__(self,name,address):
		self.name=name
		self.filename_short=get_fa_file_path("{}.fa".format(name))
		self.filename_full=os.path.abspath(self.filename_short)
		self.address=address
		self.src_dir=os.path.join(smbl.src_dir,self.name)

		if self.address[-3:]==".gz":
			self.format="gz"
		elif self.address[-5:]==".2bit":
			self.format="2bit"
		else:
			self.format="fa"

		register_fasta(self)
		
	def get(self):
		return self.filename_full

	def get_required_programs(self):
		if self.format=="2bit":
			return [smbl.prog.TWOBITTOFA]
		else:
			return []

	def install_all_steps(self):
		self.status_message("Installation started")
		self.install_pre()
		self.install()
		self.install_post()
		self.status_message("Installation completed")

	def install_pre(self):
		try:
			shutil.rmtree(self.src_dir)
		except FileNotFoundError:
			pass
		self.shell('mkdir -p "{src_dir}"'.format(src_dir=self.src_dir))

	def install(self):
		if self.format=="gz":
			archive=self.download(
				filename_full=self.abs_from_short("{}.gz".format(self.name))
			)
			self.extract_gz(
				filename_full_archive=archive,
				filename_full_file=self.filename_full,
			)
		elif self.format=="2bit":
			archive=self.download(
				filename_full=self.abs_from_short("{}.2bit".format(self.name))
			)
			self.extract_2bit(
				filename_full_archive=archive,
				filename_full_file=self.filename_full,
			)
		elif self.format=="fa":
			self.download(
				filename_full=self.filename_full
			)
		else:
			em="Unknown format '{}'".format(self.format)
			smbl.messages.error(em,program="SMBL")
			raise NotImplementedError(em)

	def install_post(self):
		self.shell('rm -fR "{src_dir}"'.format(src_dir=self.src_dir))

	def download(self,filename_full):
		self.status_message("Downloading a file: "+self.address)
		self.shell('mkdir -p "{dir}"'.format(dir=os.path.dirname(filename_full)))
		try:
			cls.shell('curl -L --insecure -o "{fn}" "{address}"'.format(fn=filename_full,address=address))
		except:
			try:
				cls.shell('curl -L --insecure -o "{fn}" "{address}"'.format(fn=filename_full,address=address))
			except:
				try:
					cls.shell('wget --no-check-certificate --output-document="{fn}" "{address}"'.format(fn=filename_full,address=address))
				except:
					cls.shell('wget --no-check-certificate --output-document="{fn}" "{address}"'.format(fn=filename_full,address=address))
		return filename_full

	def extract_gz(self,filename_full_archive,filename_full_file):
{'fraction_non_alphanumeric': 0.14793433158939204, 'fraction_numerical': 0.002886523543207649, 'mean_word_length': 4.923076923076923, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2324199', 'n_tokens_mistral': 2287, 'n_tokens_neox': 2004, 'n_words': 292}