<jupyter_start><jupyter_text>Variables<jupyter_code>stop = date.today()
start = date.today() - relativedelta(days=n_days)

# Strings for use in queries
start_date = f"{start:%Y-%m-%d}"
stop_date = f"{stop:%Y-%m-%d}"<jupyter_output><empty_output><jupyter_text>Fetch the raw data and load data we need<jupyter_code>raw_data = ga.get_activity(github_org, start_date)
bot_names = pd.read_csv('../../data/bot_names.csv')['names'].tolist()
data = raw_data.copy()

# Prepare our data
data["kind"] = data["url"].map(lambda a: "issue" if "issues/" in a else "pr")
data["mergedBy"] = data["mergedBy"].map(lambda a: a["login"] if not isinstance(a, (float, type(None))) else None)

prs = data.query("kind == 'pr'")
issues = data.query("kind == 'issue'")

# Pull out the comments
comments = []
for _, irow in data.iterrows():
    for icomment in irow['comments']['edges']:
        icomment = icomment["node"].copy()
        icomment["author"] = icomment["author"]["login"] if icomment["author"] else None
        icomment["org"] = irow["org"]
        icomment["repo"] = irow["repo"]
        icomment["id"] = irow["id"]
        comments.append(pd.Series(icomment))
comments = pd.DataFrame(comments)

# Clean up
for idata in [prs, comments, issues]:
    idata.query("author not in @bot_names", inplace=True)
    idata.drop_duplicates(subset=["url"], inplace=True)

# What are the top N repos, we will only plot these in the full data plots
top_commented_repos = comments.groupby("repo").count().sort_values("createdAt", ascending=False)['createdAt']
use_repos = top_commented_repos.head(top_n_repos).index.tolist()<jupyter_output>Running search query:
user:jupyterhub


Found 107 items, which will take 3 pages<jupyter_text>Save data to disk<jupyter_code>from pathlib import Path
path_data = Path("../data/")
path_data.mkdir(exist_ok=True)
for kind, idata in [("comments", comments), ("prs", prs), ("issues", issues)]:
    path_data = Path(f"../data/{kind}.csv")
    if path_data.exists():
        idata = pd.read_csv(path_data).append(idata)
    idata = idata.drop_duplicates(subset=["url"])
    idata.to_csv(path_data, index=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16907312529110385, 'fraction_numerical': 0.0018630647414997672, 'mean_word_length': 5.280701754385965, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '270971', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 776, 'n_tokens_neox': 759, 'n_words': 164, 'path': 'monthly_update/templates/fetch_data.ipynb', 'repo_name': 'choldgraf/jupyter-activity-snapshot', 'size': '2147'}