<jupyter_start><jupyter_text>Bodysites<jupyter_code>files=FileUtility.recursive_glob('/mounts/data/proj/asgari/github_repos/microbiomephenotype/data_config/bodysites/','*.txt')
list_of_files=[]
for file in files:
    print (file)
    list_of_files+=FileUtility.load_list(file)
list_of_files=[x+'.fsa' for x in list_of_files]
fasta_files, mapping = FileUtility.read_fasta_directory('/mounts/data/proj/asgari/dissertation/datasets/deepbio/microbiome/hmb_data/','fsa',only_files=list_of_files)
sampling_dict={6:[100,1000,5000,10000,-1]}
for k in range(6,7):
    for s in sampling_dict[k]:
        print(k)
        RS=Metagenomic16SRepresentation(fasta_files, mapping, s, 20)
        RS.generate_kmers_all(k, save='../../datasets/bodysite/'+str(k)+'-mers_rate_'+str(s))
RS=Metagenomic16SRepresentation(fasta_files, mapping, 10000, 20)
RS.generate_cpe_all(10000, save='datasets/bodysites/cpeself_10000_rate_10000', directory='datasets/bodysites/data_config/')<jupyter_output><empty_output><jupyter_text>Dental<jupyter_code>fasta_files, mapping = FileUtility.read_fasta_directory('/mounts/data/proj/asgari/dissertation/datasets/deepbio/microbiome/dental/','fastq')
sampling_dict={6:[100,1000,5000,10000,-1]}
for k in range(6,7):
    for s in sampling_dict[k]:
        print(k)
        RS=Metagenomic16SRepresentation(fasta_files, mapping, s, 20)
        RS.generate_kmers_all(k, save='../../datasets/dental/'+str(k)+'-mers_rate_'+str(s))
files=FileUtility.load_list('../../datasets/crohn/files_with_label.txt')<jupyter_output><empty_output><jupyter_text>Crohn<jupyter_code>import sys
sys.path.append('../')
from utility.file_utility import FileUtility

files=FileUtility.load_list('../../datasets/crohn/files_with_label.txt')
fasta_files, mapping = FileUtility.read_fasta_directory('/mounts/data/proj/asgari/dissertation/datasets/deepbio/microbiome/crohn/','fastq',only_files=[f.split('/')[-1] for f in files])

from make_representations.representation_maker import Metagenomic16SRepresentation

sampling_dict={6:[100,1000,5000,10000,-1]}
for k in range(6,7):
    for s in sampling_dict[k]:
        print(k)
        RS=Metagenomic16SRepresentation(fasta_files, mapping, s, 20)
        RS.generate_kmers_all(k, save='../../datasets/crohn/'+str(k)+'-mers_rate_'+str(s))
import sys
sys.path.append('../')
from utility.file_utility import FileUtility
fasta_files, mapping = FileUtility.read_fasta_directory('/mounts/data/proj/asgari/dissertation/datasets/deepbio/microbiome/crohn/','fastq')

RS=[]
RS=Metagenomic16SRepresentation(fasta_files, mapping, 10, 10)
RS.train_cpe_transformation(15000,100,'../../../datasets/deepbio/microbiome/crohn_disease/')

sampling_dict={6:[5000]}#100,1000,5000,10000,-1]}
for k in range(6,7):
    for s in sampling_dict[k]:
        print(k)
        RS=Metagenomic16SRepresentation(fasta_files, mapping, s, 20)
        RS.generate_kmers_all(k, save='../../datasets/crohn/'+str(k)+'-mers_rate_all_seq_'+str(s))
import sys
sys.path.append('../')
from utility.file_utility import FileUtility
fasta_files, mapping = FileUtility.read_fasta_directory('/mounts/data/proj/asgari/dissertation/datasets/deepbio/microbiome/crohn/','fastq')


sampling_dict={3:[20],4:[100],5:[500],6:[100,1000,2000,5000,10000,-1],7:[5000],8:[8000]}
for k in [3,4,5,7,8]:
    for s in sampling_dict[k]:
        print(k)
        RS=Metagenomic16SRepresentation(fasta_files, mapping, s, 20)
        RS.generate_kmers_all(k, save='../../datasets/crohn/'+str(k)+'-mers_rate_complete1359_seq_'+str(s))<jupyter_output>3
{'fraction_non_alphanumeric': 0.1629566210045662, 'fraction_numerical': 0.05650684931506849, 'mean_word_length': 6.066532258064516, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '315211', 'license': 'Apache-2.0', 'n_tokens_mistral': 1486, 'n_tokens_neox': 1312, 'n_words': 151, 'path': 'notebooks/.ipynb_checkpoints/Make_Representations-checkpoint.ipynb', 'repo_name': 'ehsanasgari/MicroPheno', 'size': '3504'}