<jupyter_start><jupyter_text><jupyter_code>!wget https://gist.githubusercontent.com/sbooeshaghi/6caf7976aaa21a2d378c4511def486e1/raw/5378bd2248066032fce7273e3e66cfb6f335bf82/Covid19-NPR-data.txt
import json
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
from scipy import stats

def nd(arr):
  return np.asarray(arr).reshape(-1)

def yex(ax):
    lims = [
        np.min([ax.get_xlim(), ax.get_ylim()]),  # min of both axes
        np.max([ax.get_xlim(), ax.get_ylim()]),  # max of both axes
    ]
    
    # now plot both limits against eachother
    ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0)
    ax.set_aspect('equal')
    ax.set_xlim(lims)
    ax.set_ylim(lims)
    return ax

fsize=20

plt.rcParams.update({'font.size': fsize})
%config InlineBackend.figure_format = 'retina'
with open("Covid19-NPR-data.txt", 'r') as f:
  d = json.load(f)
s = "surveillance"
cs = {"surveillance": s, "surviellence":s, "Surveillance":s}
df = pd.DataFrame(d["schools"]).drop("tot undergrad", 1).rename(columns={"most_aggressive":"testing"}).replace({"testing":cs})
df
order=["regular testing", "surveillance", "at risk", "no clear plan"][::-1]
mask = df[df["daily_three_wk_cases_per100k"].notnull()].eval("`total undergrad`>=5000")
v = df[df["daily_three_wk_cases_per100k"].notnull()][mask].groupby("testing")["daily_three_wk_cases_per100k"].apply(list)[order]
u = df[df["daily_three_wk_cases_per100k"].notnull()][mask].groupby("testing")["total undergrad"].apply(list)[order]

fig, ax = plt.subplots(figsize=(20,10), constrained_layout=True)

labels = v.index.values

data = v.values
ax.boxplot(data, whis=(0, 100), vert=False,showmeans=False)

for i in range(len(data)):
    sze = np.array(u[i])/10
    y = data[i]
    #x = np.random.normal(1+i, 0.04, size=len(y))
    x = [1+i]*len(y)
    ax.scatter(y, x, alpha=0.2,s=sze)

kwd = {
    "ylabel" : "Testing",
    "xlabel": "Average daily three week cases per 100k people",
    "yticklabels":labels
}
ax.set(**kwd)

fig.savefig("./npr.png",bbox_inches='tight', dpi=300)

fig.show()
df.query("testing=='at risk'").sort_values("daily_three_wk_cases_per100k", ascending=False).iloc[-1]
  df.iloc[df["daily_three_wk_cases_per100k"].argmax()]
df.query("`Institution Name` == 'The University of Alabama'")
df.query("`Institution Name` == 'University of Wyoming'")
  df.query("`Institution Name` == 'Tennessee Technological University'")
  df.query("`Institution Name` == 'Michigan Technological University'")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17875544986127626, 'fraction_numerical': 0.041617122473246136, 'mean_word_length': 5.555844155844156, 'pattern_counts': {'":': 7, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '183087', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1076, 'n_tokens_neox': 994, 'n_words': 171, 'path': 'notebooks/Covid19_NPR_reanalysis.ipynb', 'repo_name': 'pachterlab/COVID-19-testing-database', 'size': '2523'}