<jupyter_start><jupyter_text>possible values for detection:    - NotDetected 	    - MSMS 	    - MSMSAmbiguousPeakfinding    - MSMSIdentifiedButNotQuantified<jupyter_code>print(len(cols))<jupyter_output>364<jupyter_text>363 samples are too many to compare each-against-each. The original publication (where metaquantome uses the data from) mentions 12 paired samples, resulting in 24 samples total. For 24 samples, $24^2/2-24/2 = 276$ comparisons are required.<jupyter_code>import re
regex = re.compile(r"(\d{3}(ns|ws))", re.I)
samples = set()
count = 0
for c in cols:
    match = regex.search(c.lower())
    if match:
        count += 1
        samples.add(match.group(1))
    else:
        pass
assert count == 363, f"Expected 363 matches, got {count}"
print("\n".join(sorted(samples)))
print(len(samples), "Samples")

# load unipept results
cols = [
    "peptide",
    "GO (biological process)",
    "GO (cellular component)",
    "GO (molecular function)"
]
df_go_raw = pd.read_csv(f_pep2go, nrows=nrows, usecols=cols, index_col=0)

import math
def go_string_to_list(s, threshold=5):
    if isinstance(s, str):
        l = s.split(";")
        go_terms = []
        for go_raw in l:
            go, perc_raw = go_raw.split(" ")
            perc = int(perc_raw[1:-2])
            if perc >= threshold:
                go_terms.append(go)
        return go_terms
    else:
        return []
    
df_go = df_go_raw.applymap(go_string_to_list)
df_go.head()
df_samples.shape
def get_df_for_sample(df, sample):
    return df.filter(regex=re.compile(sample, re.I))

def get_detected_peps(df):
    s_peps = df_s.apply(lambda s: (~s.str.match("NotDetected")).any(), axis=1)
    peps = df_s[s_peps].index.tolist()
    return peps

def get_go_terms_for_peps(df_go, peps):
    present_peps = set(peps).intersection(set(df_go.index))
    diff = set(peps) - present_peps
    if diff:
        warnings.warn("The following peptides are not in the unipept table:\n" + "\n".join(diff))
    df_go_filtered = df_go.loc[present_peps, :]
    s_go_namespace = df_go_filtered.apply(lambda s: {item for sublist in s for item in sublist})
    return {i for s in s_go_namespace for i in s}

expected_count = 363
count = 0
for s in samples:
    print("Retrieving GO terms for sample:", s)
    df_s = get_df_for_sample(df_samples, s)
    count += len(df_s.columns)
    peps = get_detected_peps(df_s)
    go_terms = get_go_terms_for_peps(df_go, peps)
    f_out = os.path.join("go-per-sample", f"go_terms_{s}.csv")
    if draft:
        f_out += "_draft"
    with open(f_out, "w") as f:
        print(f"writing go terms to file: {f_out}")
        f.write(header_outfile + "\n")
        f.write("\n".join(go_terms))
    
assert count == expected_count, f"Expected {expected_count} columns to be matched, got {count}"<jupyter_output>Retrieving GO terms for sample: 769ws
{'fraction_non_alphanumeric': 0.126056338028169, 'fraction_numerical': 0.01584507042253521, 'mean_word_length': 3.418351477449456, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '371717', 'license': 'MIT', 'n_tokens_mistral': 1082, 'n_tokens_neox': 1024, 'n_words': 246, 'path': 'extract_gos.ipynb', 'repo_name': 'MEGA-GO/manuscript-data-analysis', 'size': '2840'}