<jupyter_start><jupyter_text>STEM SUBSET OF CORE<jupyter_code>%load_ext autoreload
%autoreload 2
import json
from octoflow.core import lemmatize
import pandas as pd

with open("downloads/sample_4k.json") as f:
    data = json.loads(f.read())
    
d = dict(data)
abst = [entry for entry in d["data"] if entry["abstract"]]
len(abst), abst[0]
from pydash import flatten
topics = [a["topics"] for a in abst]
journals = [a["journals"] for a in abst]<jupyter_output><empty_output><jupyter_text>Apparently CORE has journal information not as Title but as ISSN<jupyter_code>#set(flatten(topics))
STEM_stems = [
    "bio", #might be too permissive
    "phys",
    "energy",
    "engine",
    "metal",
    "surface",
    "protein",
    "medic",
    "tech",
    "compute",
    "artificial",
    "numer",
    "math",
    "chemi",
    "algorithm",
    "earth",
    "natural",
    "micro",
    "organ",
    "osis",
    "dynamics",
    "linear",
    "metabol",
    "physio",
    "electr",
    "equation",
    "graph",
    "scopy",
    "sensor",
    "agents",
    "informatic",
    "statist",
    "software",
    "meteor",
    "lipid",
    "machine",
    "vector",
    "metr",
    "icular",
    
    "neurosci",
    "dermat",
    "quantum",
    "urgy", #liturgy
    "voltage",
#    "clinical"
]
with open("stem_stems.txt", "w") as f:
    f.write("\n".join(STEM_stems))
STEM_absts = [a for a in abst if any(substring in " ".join(a['topics']).lower() for substring in STEM_stems)]
stem_topics = [a["topics"] for a in STEM_absts]

stem_terms = set(flatten(stem_topics))
all_terms = set(flatten(topics))<jupyter_output><empty_output><jupyter_text>Now let's look at our non-STEM related terms<jupyter_code>all_terms-stem_terms<jupyter_output><empty_output><jupyter_text>Other than very domain specific terms (ala phase transition) it does a decent job in weeding out the non-STEM related research topics<jupyter_code>len(STEM_absts),len(abst)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1496668375192209, 'fraction_numerical': 0.0015376729882111738, 'mean_word_length': 3.9168765743073046, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '611334', 'license': 'Apache-2.0', 'n_tokens_mistral': 758, 'n_tokens_neox': 718, 'n_words': 170, 'path': 'stem_only.ipynb', 'repo_name': 'ScienceOctopus/octopus-core', 'size': '1951'}