<jupyter_start><jupyter_text>Plot hash intersectionsusing https://pypi.org/project/UpSetPlot/docs at https://upsetplot.readthedocs.io/en/stable/<jupyter_code>from upsetplot import generate_counts, from_memberships, plot
from matplotlib import pyplot

import glob, sourmash, itertools
import pandas as pd
def load_names_df(genbank_csv, override_csv):
    "load names from auto-generated CSV, override with names in override_csv"
    names_df = pd.read_csv(genbank_csv)
    accset = set(names_df.acc)

    override_df = pd.read_csv(override_csv)
    override_df = override_df[override_df.acc.isin(accset)]
    override_df.index = range(len(override_df))

    override_df['ncbi_tax_name'] = override_df['name']

    names_df.update(override_df)
    return names_df
def hash_intersection(sigs, subtract):
    "calculate n-way hash intersection between *sigs, then remove things in 'subtract'."
    x = set(sigs[0].minhash.hashes)
    for sig in sigs[1:]:
        x.intersection_update(sig.minhash.hashes)
    return x - subtract

def get_intersections(siglist, lengths=None, transform_name=None, min_size=1):
    """generate all 1..n combinations of signatures in siglist, then calculate hash size intersections.
    
    returns a list of counts and a list of 
    
    optional 'lengths' can be the list of ns for which you want n-wise combinations, e.g. range(2, n)
    'transform_name' can be a function that takes a signature and returns a string.
    'min_size' filters intersections - only intersections >= min_size estimated bp get kept.
    """
    combos = []
    
    if not lengths:
        lengths = [ i + 1 for i in range(len(siglist)) ]
        
    for i in lengths:
        combos.extend(itertools.combinations(siglist, i))

    sofar = set()
    intersect_hashes = []
    names = []
    counts = []
    scaled = siglist[0].minhash.scaled
    for i in reversed(combos):
        ii = hash_intersection(i, sofar)
        if len(ii) * scaled < min_size:
            continue
        intersect_hashes.append(ii)
        sofar.update(ii)

        if transform_name:
            names.append( [ transform_name(ss) for ss in i ] )
        else:
            names.append( [ str(ss) for ss in i ] )
        counts.append( len(ii)*scaled )

    return counts, names<jupyter_output><empty_output><jupyter_text>example - two shewanella baltica strains + A. muciniphila<jupyter_code>sigs = glob.glob('outputs/genomes/*.sig')
siglist = [ sourmash.load_one_signature(f) for f in sigs ]

names_df = load_names_df('twofoo-acc.urls.csv', 'override-names.csv')
def get_ncbi_name(sig):
    filename = sig.filename
    acc = filename.split('_')[:2]
    acc = '_'.join(acc)
    return list(names_df[names_df.acc == acc].ncbi_tax_name)[0]

counts, names = get_intersections(siglist, lengths=range(2, len(siglist)+1),
                                  transform_name=get_ncbi_name, min_size=0)

foo = from_memberships(names, counts)
plot(foo)
#pyplot.savefig('/tmp/plot.pdf')<jupyter_output><empty_output><jupyter_text>p8808mo11 top 9 hits - any intersections?<jupyter_code>sigs = glob.glob('p8808mo11/*.sig')
siglist2 = [ sourmash.load_one_signature(f) for f in sigs ]
print(f'loaded {len(siglist2)} signatures')

names2_df = load_names_df('p8808mo11.acc.urls.csv', 'override-names.csv')


def get_ncbi_name2(sig):
    filename = sig.filename
    acc = filename.split('_')[:2]
    acc = '_'.join(acc)
    return list(names2_df[names2_df.acc == acc].ncbi_tax_name)[0]

counts, names = get_intersections(siglist2, lengths=range(2, len(siglist)+1),
                                  transform_name=get_ncbi_name2, min_size=5000)
foo = from_memberships(names, counts)
plot(foo)
#pyplot.savefig('/tmp/plot2.pdf')
counts, names = get_intersections(siglist2, lengths=range(2, len(siglist)+1),
                                  transform_name=get_ncbi_name2, min_size=0)
foo = from_memberships(names, counts)
plot(foo)
counts, names = get_intersections(siglist2,
                                  transform_name=get_ncbi_name2, min_size=10000)
foo = from_memberships(names, counts)
plot(foo)
#pyplot.savefig('/tmp/plot3.pdf')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11385061638868746, 'fraction_numerical': 0.014744984288131497, 'mean_word_length': 3.6183035714285716, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '294042', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1388, 'n_words': 301, 'path': 'hash-upsetr-example.ipynb', 'repo_name': 'mr-eyes/genome-grist', 'size': '4137'}