<commit_before>#!/usr/bin/env python
import csv
from glob import glob
import os

from shogun.utils.pickle_class import PickleClass
from shogun.taxonomy.ncbi_tree import NCBITree
from shogun import SETTINGS


class SilvaTree(PickleClass):
    def __init__(self,
                 ncbi_tree=NCBITree.load(),
                 _silva_taxdmp_dir=SETTINGS.silva_taxdmp_dir,
                 _silva_taxdmp_urls=SETTINGS.silva_taxdmp_urls):
        """

        :param ncbi_tree: NCBITree()
        :param _silva_taxdmp_dir:
        :param _silva_taxdmp_urls:
        """
        super().__init__()

        self.ncbi_tree = ncbi_tree

        self._silva_taxdmp_dir = _silva_taxdmp_dir
        self._silva_taxdmp_urls = _silva_taxdmp_urls

        self.silva_acc2taxon_id = self._parse_silva_taxonomy_file()


    def _parse_silva_taxonomy_file(self):
        silva_acc2taxon_id = {}
        files = glob(os.path.join(self._silva_taxdmp_dir, "*.txt"))
        for file in files:
            with open(file) as inf:
                csv_inf = csv.reader(inf, delimiter='\t')
                for row in csv_inf:
                    if row[0] in self.ncbi_tree.taxon_id2name:
                        silva_acc2taxon_id[row[0]] = row[-1].replace(" ", "")
        return silva_acc2taxon_id


def main():
    silva_tree = SilvaTree()
    silva_tree.save()
<commit_msg>Added the silva map tree file<commit_after>#!/usr/bin/env python
import csv
from glob import glob
import os

from shogun.utils.pickle_class import PickleClass
from shogun.taxonomy.ncbi_tree import NCBITree
from shogun import SETTINGS


class SilvaTree(PickleClass):
    def __init__(self,
                 ncbi_tree=NCBITree.load(),
                 _silva_taxdmp_dir=SETTINGS.silva_taxdmp_dir,
                 _silva_taxdmp_urls=SETTINGS.silva_taxdmp_urls):
        """

        :param ncbi_tree: NCBITree()
        :param _silva_taxdmp_dir:
        :param _silva_taxdmp_urls:
        """
        super().__init__()

        self.ncbi_tree = ncbi_tree

        self._silva_taxdmp_dir = _silva_taxdmp_dir
        self._silva_taxdmp_urls = _silva_taxdmp_urls

        self.silva_acc2taxon_id = self._parse_silva_taxonomy_file()

    def _parse_silva_taxonomy_file(self):
        silva_acc2taxon_id = {}
        files = glob(os.path.join(self._silva_taxdmp_dir, "*.txt"))
        for file in files:
            with open(file) as inf:
                csv_inf = csv.reader(inf, delimiter='\t')
                for row in csv_inf:
                    silva_acc = row[0].replace(" ", "")
                    ncbi_taxon_id = row[-1].replace(" ", "")
                    if ncbi_taxon_id in self.ncbi_tree.taxon_id2name:
                        silva_acc2taxon_id[silva_acc] = ncbi_taxon_id
        return silva_acc2taxon_id


def main():
    silva_tree = SilvaTree()
    silva_tree.save()

{'fraction_non_alphanumeric': 0.08882622488544237, 'fraction_numerical': 0.005287275290800141, 'mean_word_length': 2.3388235294117647, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3511016', 'n_tokens_mistral': 1037, 'n_tokens_neox': 995, 'n_words': 167}