<commit_before># Jython script to build the Open Tree reference taxonomy
# coding=utf-8

# Unless specified otherwise issues are in the reference-taxonomy repo:
# https://github.com/OpenTreeOfLife/reference-taxonomy/issues/...

import sys, os, csv

from org.opentreeoflife.taxa import Taxonomy, SourceTaxonomy, TsvEdits, Addition, Taxon
from org.opentreeoflife.smasher import UnionTaxonomy

# for id list
from org.opentreeoflife.taxa import CSVReader, QualifiedId
from java.io import FileReader

import ncbi_ott_assignments
sys.path.append("feed/misc/")
import adjustments, amendments
import check_inclusions
from claim import *
#import reason_report

this_source = 'https://github.com/OpenTreeOfLife/reference-taxonomy/blob/master/make-ott.py'
inclusions_path = 'inclusions.csv'
additions_clone_path = 'feed/amendments/amendments-1'
new_taxa_path = 'new_taxa'

def create_ott(version):

    ott = UnionTaxonomy.newTaxonomy('ott')
    ott.version = version;

    # Would be nice if there were tests for all of these...
    for name in names_of_interest:
        ott.eventLogger.namesOfInterest.add(name)

    ott.setSkeleton(Taxonomy.getTaxonomy('tax/separation/', 'separation'))

    # These are particularly hard cases; create alignment targets up front
    adjustments.deal_with_polysemies(ott)

    # Align and merge each source in sequence
    merge_sources(ott)

    # consider try: ... except: print '**** Exception in patch_ott'
    amendments.patch_ott(ott)

    # Remove all trees but the largest (or make them life incertae sedis)
    ott.deforestate()

    # End of topology changes.  Now assign ids.
    ids_and_additions(ott)

    # data structure integrity checks
    ott.check()

    # For deprecated id report (dump)
    ott.loadPreferredIds('ids_that_are_otus.tsv', False)
    ott.loadPreferredIds('ids_in_synthesis.tsv', True)

    return ott

def merge_sources(ott):

    # SILVA
    silva = adjustments.load_silva()
    silva_to_ott = adjustments.align_silva(silva, ott)
    align_and_merge(silva_to_ott)

    # Hibbett 2007
    h2007 = adjustments.load_h2007()
    h2007_to_ott = ott.alignment(h2007)
    align_and_merge(h2007_to_ott)

    # Index Fungorum
    fungorum = adjustments.load_fung()
    (fungi, fungorum_sans_fungi) = split_taxonomy(fungorum, 'Fungi')
    align_and_merge(adjustments.align_fungi(fungi, ott))

    # the non-Fungi from Index Fungorum get absorbed below

    lamiales = adjustments.load_lamiales()
    align_and_merge(adjustments.align_lamiales(lamiales, ott))

    # WoRMS
    # higher priority to Worms for Malacostraca, Cnidaria so we split out
    # those clades from worms and absorb them before NCBI
    worms = adjustments.load_worms()
    # Malacostraca instead of Decapoda because M. is in the separation taxonomy
    (malacostraca, worms_sans_malacostraca) = split_taxonomy(worms, 'Malacostraca')
    align_and_merge(ott.alignment(malacostraca))
    (cnidaria, low_priority_worms) = split_taxonomy(worms_sans_malacostraca, 'Cnidaria')
    align_and_merge(ott.alignment(cnidaria))

    # NCBI
    ncbi = adjustments.load_ncbi()

    # analyzeOTUs sets flags on questionable taxa (hybrid, metagenomes,
    #  etc) to allow the option of suppression downstream
    ncbi.analyzeOTUs()

    ncbi_to_ott = adjustments.align_ncbi(ncbi, silva, ott)
    align_and_merge(ncbi_to_ott)

    # Reporting
    # Get mapping from NCBI to OTT, derived via SILVA and Genbank.
    mappings = load_ncbi_to_silva(ncbi, silva, silva_to_ott)
    compare_ncbi_to_silva(mappings, silva_to_ott)

    debug_divisions('Reticularia splendens', ncbi, ott)

    # Low-priority WoRMS<commit_msg>use worms for Mollusca<commit_after># Jython script to build the Open Tree reference taxonomy
# coding=utf-8

# Unless specified otherwise issues are in the reference-taxonomy repo:
# https://github.com/OpenTreeOfLife/reference-taxonomy/issues/...

import sys, os, csv

from org.opentreeoflife.taxa import Taxonomy, SourceTaxonomy, TsvEdits, Addition, Taxon
from org.opentreeoflife.smasher import UnionTaxonomy

# for id list
from org.opentreeoflife.taxa import CSVReader, QualifiedId
from java.io import FileReader

import ncbi_ott_assignments
sys.path.append("feed/misc/")
import adjustments, amendments
import check_inclusions
from claim import *
#import reason_report

this_source = 'https://github.com/OpenTreeOfLife/reference-taxonomy/blob/master/make-ott.py'
inclusions_path = 'inclusions.csv'
additions_clone_path = 'feed/amendments/amendments-1'
new_taxa_path = 'new_taxa'

def create_ott(version):

    ott = UnionTaxonomy.newTaxonomy('ott')
    ott.version = version;

    # Would be nice if there were tests for all of these...
    for name in names_of_interest:
        ott.eventLogger.namesOfInterest.add(name)

    ott.setSkeleton(Taxonomy.getTaxonomy('tax/separation/', 'separation'))

    # These are particularly hard cases; create alignment targets up front
    adjustments.deal_with_polysemies(ott)

    # Align and merge each source in sequence
    merge_sources(ott)

    # consider try: ... except: print '**** Exception in patch_ott'
    amendments.patch_ott(ott)

    # Remove all trees but the largest (or make them life incertae sedis)
    ott.deforestate()

    # End of topology changes.  Now assign ids.
    ids_and_additions(ott)

    # data structure integrity checks
    ott.check()

    # For deprecated id report (dump)
    ott.loadPreferredIds('ids_that_are_otus.tsv', False)
    ott.loadPreferredIds('ids_in_synthesis.tsv', True)

    return ott

def merge_sources(ott):

    # SILVA
    silva = adjustments.load_silva()
    silva_to_ott = adjustments.align_silva(silva, ott)
    align_and_merge(silva_to_ott)

    # Hibbett 2007
    h2007 = adjustments.load_h2007()
    h2007_to_ott = ott.alignment(h2007)
    align_and_merge(h2007_to_ott)

    # Index Fungorum
    fungorum = adjustments.load_fung()
    (fungi, fungorum_sans_fungi) = split_taxonomy(fungorum, 'Fungi')
    align_and_merge(adjustments.align_fungi(fungi, ott))

    # the non-Fungi from Index Fungorum get absorbed below

    lamiales = adjustments.load_lamiales()
    align_and_merge(adjustments.align_lamiales(lamiales, ott))

    # WoRMS
    # higher priority to Worms for Malacostraca, Cnidaria, Mollusca
    #  so we split out
    # those clades from worms and absorb them before NCBI
    worms = adjustments.load_worms()
    # Malacostraca instead of Decapoda because M. is in the separation taxonomy
    (malacostraca, worms_sans_malacostraca) = split_taxonomy(worms, 'Malacostraca')
    align_and_merge(ott.alignment(malacostraca))
    (cnidaria, worms_sans_cnidaria) = split_taxonomy(worms_sans_malacostraca, 'Cnidaria')
    align_and_merge(ott.alignment(cnidaria))
    (mollusca, low_priority_worms) = split_taxonomy(worms_sans_cnidaria, 'Mollusca')
    align_and_merge(ott.alignment(mollusca))

    # NCBI
    ncbi = adjustments.load_ncbi()

    # analyzeOTUs sets flags on questionable taxa (hybrid, metagenomes,
    #  etc) to allow the option of suppression downstream
    ncbi.analyzeOTUs()

    ncbi_to_ott = adjustments.align_ncbi(ncbi, silva, ott)
    align_and_merge(ncbi_to_ott)

    # Reporting
    # Get mapping from NCBI to OTT, derived via SILVA and Genbank.
    mappings = load_ncbi_to_silva(ncbi, silva, silva_to_ott)
    compare_ncbi_to_silva(mappings, silva_to_ott)

    debug_divisions('Reticularia splendens', ncbi, ott)

    # Low-priority WoRMS
{'fraction_non_alphanumeric': 0.08456031338646494, 'fraction_numerical': 0.007024179386735108, 'mean_word_length': 4.2585227272727275, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5725894', 'n_tokens_mistral': 2668, 'n_tokens_neox': 2471, 'n_words': 586}