<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.api as sm
%config InlineBackend.figure_format = "svg"<jupyter_output>/usr/local/lib/python3.7/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
  import pandas.util.testing as tm<jupyter_text>In this notebook we recreate Figure 2 (A) using the original data from Venable 2007.<jupyter_code>data = pd.read_csv("http://www.eebweb.arizona.edu/faculty/venable/LTREB/species_x_year.csv",delimiter=",")
data

# get a list of all the species in the data set
species = data.species.unique()

def getxy(species):
  """
  This function returns the variation in reproductive success and the 
  fraction of germinating seeds, which will be the x and y coordinate in the plot
  """
  df = data[data.species == species]
  frac = np.array([float(x) if x != "." else 0 for x in df["germ fraction"].values if x != "."])
  S = np.array([float(x) for x in df["bx"].values if x != "."])
  y = np.mean(frac)
  x = np.exp(np.std(np.log(S)))
  return [x,y]
# now we get the x and y coordinates for all species
x,y = np.array([getxy(s) for s in species]).transpose();
fig,ax = plt.subplots(figsize=(5,4))
for k in range(len(species)):
  ax.plot(x[k],y[k],'o',fillstyle="none")
  ax.text(x[k],y[k]+0.05,species[k])
ax.set_xlabel("standard deviation of reproductive success")
ax.set_ylabel("germination fraction")
ax.set_ylim(0,1)
ax.set_xlim(2,6);<jupyter_output><empty_output><jupyter_text>Now let's perform a linear regression and confirm that there is indeed a statistically signficant correlation.<jupyter_code>X = sm.add_constant((x - np.mean(x))/np.std(x))
model= sm.OLS((y - np.mean(y))/np.std(y),X) 
results = model.fit()
b_fit,a_fit = results.params
# now let's plot it and compare to the data
fig,ax = plt.subplots(figsize=(5,2))
ax.plot((x - np.mean(x))/np.std(x),(y - np.mean(y))/np.std(y),"o",label="noisy measurments")
ax.plot((x - np.mean(x))/np.std(x),a_fit*(x - np.mean(x))/np.std(x) + b_fit,"-",label="fit")
ax.set_xlabel("x")
ax.set_ylabel("y")
ax.legend()
print(results.summary())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16184971098265896, 'fraction_numerical': 0.009337483325922631, 'mean_word_length': 5.232686980609419, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '408234', 'license': 'MIT', 'n_tokens_mistral': 867, 'n_tokens_neox': 826, 'n_words': 204, 'path': 'desert_annuals.ipynb', 'repo_name': 'elevien/phenotypic_variability_review', 'size': '2249'}