<jupyter_start><jupyter_text>Data Preprocessing  Demographics Data From Figshare (WIP)<jupyter_code># Demographics-Combined200411.csv,https://ndownloader.figshare.com/files/22345605
figshare_url = "https://ndownloader.figshare.com/files/22345605"

content = requests.get(figshare_url).content
# df_dm_combined = pd.read_csv(io.StringIO(content.decode('utf-8')))

# Demographics-CombinedByCountry200411.csv,https://ndownloader.figshare.com/files/22345602
figshare_url = "https://ndownloader.figshare.com/files/22345602"

content = requests.get(figshare_url).content
# df_dm = pd.read_csv(io.StringIO(content.decode('utf-8')))

# df_dm.head()<jupyter_output><empty_output><jupyter_text>Required Setups- All datasets should be placed in `../data/combined` (e.g., `../data/combined/Demographics-Combinedyymmdd.csv`).<jupyter_code># Path to save *.PNG files
SAVE_DIR = join("..", "output")

# Country Info
ALL_COUNTRY = "All countries"
ALL_COUNTRY_COLOR = "#444444"
COUNTRIES = [ "France", "Germany", "Italy", "Singapore", "USA" ]
COUNTRY_COLOR = [ "#0072B2", "#E69F00", "#009E73", "#CC79A7", "#D55E00" ]
COLOR_BY_COUNTRY = { COUNTRIES[i]: COUNTRY_COLOR[i] for i in range(len(COUNTRIES)) }<jupyter_output><empty_output><jupyter_text>Data Preprocess<jupyter_code>def preprocess_demo_df(df_dm):
    
    # Drop unused columns before preprocessing for the simplicity
    df_dm = df_dm.drop(columns=[
        COLUMNS.UNMASKED_SITES_TOTAL_PATIENTS,
        COLUMNS.UNMASKED_SITES_AGE_0TO2,
        COLUMNS.UNMASKED_SITES_AGE_3TO5,
        COLUMNS.UNMASKED_SITES_AGE_6TO11,
        COLUMNS.UNMASKED_SITES_AGE_12TO17,
        COLUMNS.UNMASKED_SITES_AGE_18TO25,
        COLUMNS.UNMASKED_SITES_AGE_26TO49,
        COLUMNS.UNMASKED_SITES_AGE_50TO69,
        COLUMNS.UNMASKED_SITES_AGE_70TO79,
        COLUMNS.UNMASKED_SITES_AGE_80PLUS,
        COLUMNS.MASKED_SITES_TOTAL_PATIENTS,
        COLUMNS.MASKED_SITES_AGE_0TO2,
        COLUMNS.MASKED_SITES_AGE_3TO5,
        COLUMNS.MASKED_SITES_AGE_6TO11,
        COLUMNS.MASKED_SITES_AGE_12TO17,
        COLUMNS.MASKED_SITES_AGE_18TO25,
        COLUMNS.MASKED_SITES_AGE_26TO49,
        COLUMNS.MASKED_SITES_AGE_50TO69,
        COLUMNS.MASKED_SITES_AGE_70TO79,
        COLUMNS.MASKED_SITES_AGE_80PLUS,
        COLUMNS.MASKED_UPPER_BOUND_TOTAL_PATIENTS,
        COLUMNS.TOTAL_PATIENTS,
    ])

    # Wide to long
    df_dm = pd.melt(df_dm, id_vars=[
        COLUMNS.SITE_ID,
        COLUMNS.SEX,
        COLUMNS.MASKED_UPPER_BOUND_AGE_0TO2,
        COLUMNS.MASKED_UPPER_BOUND_AGE_3TO5,
        COLUMNS.MASKED_UPPER_BOUND_AGE_6TO11,
        COLUMNS.MASKED_UPPER_BOUND_AGE_12TO17,
        COLUMNS.MASKED_UPPER_BOUND_AGE_18TO25,
        COLUMNS.MASKED_UPPER_BOUND_AGE_26TO49,
        COLUMNS.MASKED_UPPER_BOUND_AGE_50TO69,
        COLUMNS.MASKED_UPPER_BOUND_AGE_70TO79,
        COLUMNS.MASKED_UPPER_BOUND_AGE_80PLUS,
    ])
    df_dm = df_dm.rename(columns={"variable": COLUMNS.AGE_GROUP, "value": COLUMNS.NUM_PATIENTS})

    df_dm[COLUMNS.SEX] = df_dm[COLUMNS.SEX].apply(lambda x: x.capitalize())

    # Drop unused columns
    df_dm = df_dm.drop(columns=[
        COLUMNS.MASKED_UPPER_BOUND_AGE_0TO2,
        COLUMNS.MASKED_UPPER_BOUND_AGE_3TO5,
        COLUMNS.MASKED_UPPER_BOUND_AGE_6TO11,
        COLUMNS.MASKED_UPPER_BOUND_AGE_12TO17,
        COLUMNS.MASKED_UPPER_BOUND_AGE_18TO25,
        COLUMNS.MASKED_UPPER_BOUND_AGE_26TO49,
        COLUMNS.MASKED_UPPER_BOUND_AGE_50TO69,
        COLUMNS.MASKED_UPPER_BOUND_AGE_70TO79,
        COLUMNS.MASKED_UPPER_BOUND_AGE_80PLUS,
    ])

    # Add a percentage column
    unique_site_ids = df_dm[COLUMNS.SITE_ID].unique()
    for site in unique_site_ids:
        unique_sex = df_dm[df_dm[COLUMNS.SITE_ID] == site][COLUMNS.SEX].unique()
        for sex in unique_sex:
            df_filter = (df_dm[COLUMNS.SITE_ID] == site) & (df_dm[COLUMNS.SEX] == sex)
            total = df_dm.loc[
                df_filter, 
                COLUMNS.NUM_PATIENTS
            ].sum()
            
            df_dm.loc[
                df_filter, 
                "per_patients"
            ] = df_dm.loc[df_filter, COLUMNS.NUM_PATIENTS] / total * 100
    
    # Use readable names
    df_dm.loc[df_dm[COLUMNS.SITE_ID] == "Combined", COLUMNS.SITE_ID] = ALL_COUNTRY
    readable_age_group = {
        COLUMNS.AGE_0TO2: "0 - 2",
        COLUMNS.AGE_3TO5: "3 - 5",
        COLUMNS.AGE_6TO11: "6 - 11",
        COLUMNS.AGE_12TO17: "12 - 17",
        COLUMNS.AGE_18TO25: "18 - 25",
        COLUMNS.AGE_26TO49: "26 - 49",
        COLUMNS.AGE_50TO69: "50 - 69",
        COLUMNS.AGE_70TO79: "70 - 79",
        COLUMNS.AGE_80PLUS: "80+"
    }
    df_dm[COLUMNS.AGE_GROUP] = df_dm[COLUMNS.AGE_GROUP].apply(lambda x: readable_age_group[x])

    return df_dm

# Process data
df_dm = read_combined_by_country_demographics_df()
df_dm = preprocess_demo_df(df_dm)

df_dm_combined = read_combined_demographics_df()
df_dm_combined = preprocess_demo_df(df_dm_combined)

# Merge
df_dm = pd.concat([df_dm, df_dm_combined])

df_dm<jupyter_output><empty_output><jupyter_text>Visualizations<jupyter_code>def demographics(is_percent=False, by_country=False):

    # Selection components
    country_dropdown = alt.binding_select(options=[ALL_COUNTRY] + COUNTRIES)
    country_selection = alt.selection_single(fields=[COLUMNS.SITE_ID], bind=country_dropdown, name="Country", init={COLUMNS.SITE_ID: ALL_COUNTRY})
    sex_dropdown = alt.binding_select(options=["All", "Male", "Female"])
    sex_selection = alt.selection_single(fields=[COLUMNS.SEX], bind=sex_dropdown, name="Sex", init={COLUMNS.SEX: "All"})
    color_field = COLUMNS.SITE_ID if by_country else COLUMNS.SEX
    legend_selection = alt.selection_multi(fields=[color_field], bind="legend")

    # Filter
    filtered_chart = alt.Chart(df_dm).transform_filter(
        alt.datum[COLUMNS.SEX] != "Other"
    ).transform_filter(
        legend_selection
    )

    if by_country:
        filtered_chart = filtered_chart.transform_filter(
            sex_selection
        ).transform_filter(
            alt.datum[COLUMNS.SITE_ID] != ALL_COUNTRY
        )
    else:
        filtered_chart = filtered_chart.transform_filter(
            country_selection
        ).transform_filter(
            alt.datum[COLUMNS.SEX] != "All"
        )

    DEMO_TOOLTIP = [
        alt.Tooltip(COLUMNS.SITE_ID, title="Country"),
        alt.Tooltip(COLUMNS.SEX, title="Sex"),
        alt.Tooltip(COLUMNS.AGE_GROUP, title="Age group"),
        alt.Tooltip(COLUMNS.NUM_PATIENTS, title="Number of patients")
    ]

    y_field = "per_patients" if is_percent else COLUMNS.NUM_PATIENTS
    if is_percent:
        DEMO_TOOLTIP += [alt.Tooltip("per_patients", title="Percentage of patients (%)", format=".1f")]

    # Render
    color_scale = alt.Scale(domain=COUNTRIES, range=COUNTRY_COLOR)  if by_country else alt.Scale(domain=["Male", "Female"], range=["#3366cc", "#dc3912"]) 
    y_title = "Percentage of patients (%)" if is_percent else "Number of patients"
    bar = filtered_chart.mark_bar().encode(
        x=alt.X(f"{color_field}:N", title=None, axis=None),
        y=alt.Y(f"{y_field}:Q", title=y_title, axis=alt.Axis(tickCount=5)),
        color=alt.Color(f"{color_field}:N", title=None, scale=color_scale),
        tooltip=DEMO_TOOLTIP
    ).properties(width=67,height=400)

    result_vis = bar.encode(
        column=alt.Column(
            "age_group:O",
            sort=["age_0to2","age_3to5","age_6to11","age_12to17","age_18to25","age_26to49","age_50to69","age_70to79", "age_80plus"],
            header=alt.Header(labelOrient="bottom", title="Age group", titleOrient="bottom")
        )
    ).add_selection(
        legend_selection
    )

    if by_country:
        result_vis = result_vis.add_selection(
            sex_selection
        )
    else:
        result_vis = result_vis.add_selection(
            country_selection
        )

    return result_vis<jupyter_output><empty_output><jupyter_text>Demographics by sex<jupyter_code>demo = apply_theme(demographics(is_percent=True), legend_stroke_color="lightgray", axis_title_font_size=18).properties(title={
    "text": "Demographics by Sex",
    "subtitle": get_visualization_subtitle(),
    "subtitleColor": "gray",
    "anchor": "start",
    "dx": 60
})
demo.display()

for_website(demo, "Demographics", "Demographics by sex") # TODO: Remove this before deploying notebook
# save(demo, join(SAVE_DIR, f"demographics.png".lower()), scalefactor=2.0) # Uncomment this to save *.png files<jupyter_output><empty_output><jupyter_text>Demographics by country<jupyter_code>demo = apply_theme(demographics(is_percent=True, by_country=True), legend_stroke_color="lightgray").properties(title={
    "text": "Demographics by Country",
    "subtitle": get_visualization_subtitle(),
    "subtitleColor": "gray",
    "anchor": "start",
    "dx": 60
})
demo.display()

for_website(demo, "Demographics", "Demographics by country") # TODO: Remove this before deploying notebook
# save(demo, join(SAVE_DIR, f"demographics.png".lower()), scalefactor=2.0) # Uncomment this to save *.png files<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11719181868435599, 'fraction_numerical': 0.03360972913211719, 'mean_word_length': 3.7535470310036785, 'pattern_counts': {'":': 12, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '128387', 'license': 'MIT', 'n_tokens_mistral': 3632, 'n_tokens_neox': 3414, 'n_words': 445, 'path': 'notebooks/01_demographics_altair.ipynb', 'repo_name': 'upsetjs/visualization-notebooks', 'size': '9045'}