<commit_before>#!/usr/bin/env python


"""
heatsequer experiment input/output functions
"""

# amnonscript

__version__ = "0.9"

import heatsequer as hs

import numpy as np
import csv
import biom
import os
from pdb import set_trace as XXX
import hashlib


def load(tablename, mapname='map.txt', taxfile='', nameisseq=True,studyname=False,tabletype='biom',normalize=True,addsname='',keepzero=False,removefrom=False,removenum=1,mapsampletolowercase=False,sortit=True):
	"""
	Load an experiment - a biom table and a mapping file
	input:
	tablename - the name of the biom table file
	mapname - name of the mapping file
	taxfile - empty ('') to load taxonomy from biom table, non-empty to load
	from rdp output file (web)
	nameisseq - False to keep otu name as sid without hashing it, True to treat otuid as sequence
	addsname - a string to add to each table sample name (or empty to not add)
	studyname - Flase to assign from table file name, otherwise string to store as study name
	tabletype:
		'biom' - a biom table
		'meta' - a metabolomics table (row per sample, col per metabolite, can contain duplicate metaboliteids)
	normalize - True to normalize to 10k reads per sample, False to not normalize (change to mean 10k reads/sample)
	keepzero : bool
		True (default) to keep samples with 0 reads, False to throw away
	removefrom : string
		if non empty - cut table sample name after (and including) the first occurance of removefrom
	mapsampletolowercase : bool
		True to convert the mapping file sample id to lower case (for EMP data). default=False
	sortit : bool
		True (default) to sort sequences by taxonomy, False to not sort
	output:
	an experiment class for the current experiment
	"""

	params=locals()

	# load the table
	if tabletype=='biom':
		hs.Debug(6,'Loading biom table')
		table=biom.load_table(tablename)
	elif tabletype=='meta':
		hs.Debug(6,'Loading metabolite table')
		table=loadmetabuckettable(tablename)
	else:
		hs.Debug(9,'Table type %s not supported' % tabletype)
		return False

	datamd5=hashlib.md5()
	datamd5.update(str(table.matrix_data.todense().A).encode('utf=8'))
	datamd5=datamd5.hexdigest()
	# if need to cut table sample names
	if removefrom:
		idtable={}
		foundids={}
		ids=table.ids(axis='sample')
		for cid in ids:
			if removefrom in cid:
#				tid=cid[:cid.find(removefrom)]
				fpos=hs.findn(cid,removefrom,removenum)
				if fpos==-1:
					hs.Debug(6,'Did not find enough %s in %s' % (removefrom,cid))
					tid=cid
				else:
					tid=cid[:fpos]
			else:
				hs.Debug(6,'%s not found in sample name %s (removefrom)' % (removefrom,cid))
				tid=cid
			if tid in foundids:
				hs.Debug(6,'already have id %s' % cid)
				idtable[cid]=tid+str(foundids[tid])
				foundids[tid]+=1
			else:
				foundids[tid]=1
				idtable[cid]=tid
		table=table.update_ids(idtable,axis='sample')

	# if need to add constant string to sample names in table
	if addsname!='':
		idtable={}
		ids=table.ids(axis='sample')
		for cid in ids:
			idtable[cid]=addsname+cid
		table=table.update_ids(idtable,axis='sample')


	smap = {}
	mapsamples = []
	if mapname:
		# if mapping file supplied, load it
		mapsamples,smap,fields,mapmd5=loadmap(mapname,mapsampletolowercase=mapsampletolowercase)
	else:
		# no mapping file, so just create the #SampleID field
		hs.Debug(6,'No mapping file supplied - using just sample names')
		tablesamples = table.ids(axis='sample')
		for cid in tablesamples:
			smap[cid]={'#SampleID':cid}
			mapsamples.append(cid)<commit_msg>fix mapmd5 when no map file supplied again<commit_after>#!/usr/bin/env python


"""
heatsequer experiment input/output functions
"""

# amnonscript

__version__ = "0.9"

import heatsequer as hs

import numpy as np
import csv
import biom
import os
from pdb import set_trace as XXX
import hashlib


def load(tablename, mapname='map.txt', taxfile='', nameisseq=True,studyname=False,tabletype='biom',normalize=True,addsname='',keepzero=False,removefrom=False,removenum=1,mapsampletolowercase=False,sortit=True):
	"""
	Load an experiment - a biom table and a mapping file
	input:
	tablename - the name of the biom table file
	mapname - name of the mapping file
	taxfile - empty ('') to load taxonomy from biom table, non-empty to load
	from rdp output file (web)
	nameisseq - False to keep otu name as sid without hashing it, True to treat otuid as sequence
	addsname - a string to add to each table sample name (or empty to not add)
	studyname - Flase to assign from table file name, otherwise string to store as study name
	tabletype:
		'biom' - a biom table
		'meta' - a metabolomics table (row per sample, col per metabolite, can contain duplicate metaboliteids)
	normalize - True to normalize to 10k reads per sample, False to not normalize (change to mean 10k reads/sample)
	keepzero : bool
		True (default) to keep samples with 0 reads, False to throw away
	removefrom : string
		if non empty - cut table sample name after (and including) the first occurance of removefrom
	mapsampletolowercase : bool
		True to convert the mapping file sample id to lower case (for EMP data). default=False
	sortit : bool
		True (default) to sort sequences by taxonomy, False to not sort
	output:
	an experiment class for the current experiment
	"""

	params=locals()

	# load the table
	if tabletype=='biom':
		hs.Debug(6,'Loading biom table')
		table=biom.load_table(tablename)
	elif tabletype=='meta':
		hs.Debug(6,'Loading metabolite table')
		table=loadmetabuckettable(tablename)
	else:
		hs.Debug(9,'Table type %s not supported' % tabletype)
		return False

	datamd5=hashlib.md5()
	datamd5.update(str(table.matrix_data.todense().A).encode('utf=8'))
	datamd5=datamd5.hexdigest()
	# if need to cut table sample names
	if removefrom:
		idtable={}
		foundids={}
		ids=table.ids(axis='sample')
		for cid in ids:
			if removefrom in cid:
#				tid=cid[:cid.find(removefrom)]
				fpos=hs.findn(cid,removefrom,removenum)
				if fpos==-1:
					hs.Debug(6,'Did not find enough %s in %s' % (removefrom,cid))
					tid=cid
				else:
					tid=cid[:fpos]
			else:
				hs.Debug(6,'%s not found in sample name %s (removefrom)' % (removefrom,cid))
				tid=cid
			if tid in foundids:
				hs.Debug(6,'already have id %s' % cid)
				idtable[cid]=tid+str(foundids[tid])
				foundids[tid]+=1
			else:
				foundids[tid]=1
				idtable[cid]=tid
		table=table.update_ids(idtable,axis='sample')

	# if need to add constant string to sample names in table
	if addsname!='':
		idtable={}
		ids=table.ids(axis='sample')
		for cid in ids:
			idtable[cid]=addsname+cid
		table=table.update_ids(idtable,axis='sample')


	smap = {}
	mapsamples = []
	mapmd5=''
	if mapname:
		# if mapping file supplied, load it
		mapsamples,smap,fields,mapmd5=loadmap(mapname,mapsampletolowercase=mapsampletolowercase)
	else:
		# no mapping file, so just create the #SampleID field
		hs.Debug(6,'No mapping file supplied - using just sample names')
		tablesamples = table.ids(axis='sample')
		for cid in tablesamples:
			smap[cid]={'#SampleID':cid}
			mapsamples.append(cid)
{'fraction_non_alphanumeric': 0.10463784712281707, 'fraction_numerical': 0.007443458345261953, 'mean_word_length': 4.407894736842105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5257257', 'n_tokens_mistral': 2551, 'n_tokens_neox': 2333, 'n_words': 761}