<jupyter_start><jupyter_text>Preparation<jupyter_code>import datetime as dt
import re
import json
import time
import pandas as pd
import numpy as np
# An arbitrary integer to use as a sampling seed so the sampling is deterministic and can be re-run
RANDOM_STATE = 13440

frame = pd.read_parquet("secrets/sampling-frame.parquet")
strata_targets = pd.read_table("definitions/strata-sample-targets.tsv").set_index("project_group")<jupyter_output><empty_output><jupyter_text>Set sample sizes<jupyter_code># Stratify sample
strata_sizes = frame.groupby(["project_group", "edit_bin"]).size().unstack()

# `strata_targets` has string column names, but `strata_sizes` has categoricals
strata_targets.columns = strata_sizes.columns
# The targets must not be larger than the strata, or the sampling will hit an error
excessive_targets = (strata_targets > strata_sizes)
assert excessive_targets.any(axis=None) == False
# Set target to 0 for any strata with fewer than 10 users,
# to prevent cases where de-anonymization of data would be too easy
strata_sizes = strata_targets.where(strata_sizes >= 10, other=0)
strata = (
  strata_sizes
  .unstack()
  .reset_index()
  .rename({0: "sample_size"}, axis=1)
  # Ensure columns are ordered in the expected way
  [["project_group", "edit_bin", "sample_size"]]
)

strata.head()<jupyter_output><empty_output><jupyter_text>Pull samples<jupyter_code>def sample_stratum(project_group, edit_bin, sample_size):
  """
  returns the sampled users as a data frame
  """
  stratum = frame.query("project_group == @project_group & edit_bin == @edit_bin")
  sample = stratum.sample(n=sample_size, random_state=RANDOM_STATE)
  return sample

f = []

for row in strata.itertuples(index=False):
  f.append(sample_stratum(*row))
  
sampled_users = pd.concat(f)<jupyter_output><empty_output><jupyter_text>Save samples table<jupyter_code>sampled_users.to_csv("secrets/sampled_users.tsv", sep="\t")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10710607621009269, 'fraction_numerical': 0.006694129763130793, 'mean_word_length': 6.58984375, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '218104', 'license': 'MIT', 'n_tokens_mistral': 689, 'n_tokens_neox': 655, 'n_words': 161, 'path': '03-sampling.ipynb', 'repo_name': 'wikimedia-research/Community-Engagement-Insights', 'size': '1942'}