<jupyter_start><jupyter_text>Observations and Insights<jupyter_code># 3 Observations
#----------------------
#1 - Lineplot
#    The Capomulin Drug Regimen Average Tumor Volume (mm3) size shrunk from just over 1,100 (mm3) to just
#    under 800 (mm3) between Timepoints 0 and 45.
#----------------------
#2 - Boxplot
#    The Ramicane & Capomulin Drug Regimens yield better results than the Ceftamin and Ifubinol Drug Regimens
#    based on the Boxplot chart.  Ramicane's median point is more centered (ie consistent) than Capomulin,
#    while Capomulin's box (1st & 3rd quartiles) are a tighter fit.
#----------------------
#3 - Correlation & Regression
#    The Capomulin Drug Regimen has an 84.19% correlation between Weight (g) and Tumor Volume (mm3).  This
#    is a strong correlation.  The slope is 95.44% Tumor Volume (mm3) to Weight (g) with a Tumor Volume (mm3)
#    y-intercept constant of 21.55 (mm3).

# Dependencies and Setup
import matplotlib.pyplot as plt
import pandas as pd
import scipy.stats as st
import numpy as np
from scipy.stats import linregress

# Study data files
mouse_metadata_path = "data/Mouse_metadata.csv"
study_results_path = "data/Study_results.csv"

# Read the mouse data and the study results
mouse_metadata = pd.read_csv(mouse_metadata_path)
study_results = pd.read_csv(study_results_path)

#Combine the data into a single dataset
study_data_complete = pd.merge(study_results, mouse_metadata, on="Mouse ID")

# Display the data table for preview
study_data_complete.head()

# Checking the number of mice.
mouse_count = len(study_data_complete["Mouse ID"].unique())
mouse_count
# Getting the duplicate mice by ID number that shows up for Mouse ID and Timepoint. 
dup_entries = study_data_complete.duplicated(subset=["Mouse ID", "Timepoint"])
dup_mouse_ids = study_data_complete.loc[dup_entries, "Mouse ID"].unique()

#AskBCS Learning Assistant
# Optional: Get all the data for the duplicate mouse ID. 
dropna_records = study_data_complete.loc[dup_entries]
dropna_records
# Create a clean DataFrame by dropping the duplicate mouse by its ID.
is_dup_entry = study_data_complete['Mouse ID'].isin(dup_mouse_ids)
not_duplicated_entries = is_dup_entry == False
clean_study_data_complete = study_data_complete[not_duplicated_entries]

#is_dup_entry.count()
clean_study_data_complete.count()

#AskBCS Learning Assistant
#is_dup_entry = study_data_complete['Mouse ID'].isin(duplicate_mouse_ids)
#not_duplicated_entries = is_dup_entry == False
#clean_study_data_complete = study_data_complete[not_duplicated_entries]
# Checking the number of mice in the clean DataFrame.
mouse_count = len(clean_study_data_complete["Mouse ID"].unique())
mouse_count<jupyter_output><empty_output><jupyter_text>Summary Statistics<jupyter_code># Generate a summary statistics table of mean, median, variance, standard deviation, and SEM of the tumor volume for each regimen

# This method is the most straighforward, creating multiple series and putting them all together at the end.
drug_regimen_ds = clean_study_data_complete.groupby(["Drug Regimen"])
drug_regimen_ds["Tumor Volume (mm3)"].describe()

#https://stackoverflow.com/questions/33575587/pandas-dataframe-how-to-apply-describe-to-each-group-and-add-to-new-columns
#mouse_qry_describe = pd.DataFrame(group.describe().rename(columns={'score':name}).squeeze() for name, group in df.groupby('name'))
# Generate a summary statistics table of mean, median, variance, standard deviation, and SEM of the tumor volume for each regimen

# This method produces everything in a single groupby function
drug_regimen_ds = clean_study_data_complete.groupby(["Drug Regimen"])
drug_regimen_ds["Tumor Volume (mm3)"].describe()<jupyter_output><empty_output><jupyter_text>Bar and Pie Charts<jupyter_code># Generate a bar plot showing the total number of mice for each treatment throughout the course of the study using pandas. 

##Add series values
drug_regimen_items=clean_study_data_complete["Drug Regimen"].value_counts()

##Add Value Labels
##https://stackoverflow.com/questions/28931224/adding-value-labels-on-a-matplotlib-bar-chart
# Plot the figure.
plt.figure(figsize=(12, 8))
ax = drug_regimen_items.plot(kind='bar')
ax.set_title('Drug Regimen Data Points')
ax.set_xlabel('Drug Regimen')
ax.set_ylabel('Number of Data Points')


def add_value_labels(ax, spacing=5):
    """Add labels to the end of each bar in a bar chart.

    Arguments:
        ax (matplotlib.axes.Axes): The matplotlib object containing the axes
            of the plot to annotate.
        spacing (int): The distance between the labels and the bars.
    """

    # For each bar: Place a label
    for rect in ax.patches:
        # Get X and Y placement of label from rect.
        y_value = rect.get_height()
        x_value = rect.get_x() + rect.get_width() / 2

        # Number of points between bar and label. Change to your liking.
        space = spacing
        # Vertical alignment for positive values
        va = 'bottom'

        # If value of bar is negative: Place label below bar
        if y_value < 0:
            # Invert space to place label below
            space *= -1
            # Vertically align label at top
            va = 'top'

        # Use Y value as label and format number with one decimal place
        label = "{:.1f}".format(y_value)

        # Create annotation
        ax.annotate(
            label,                      # Use `label` as label
            (x_value, y_value),         # Place label at end of the bar
            xytext=(0, space),          # Vertically shift label by `space`
            textcoords="offset points", # Interpret `xytext` as offset in points
            ha='center',                # Horizontally center label
            va=va)                      # Vertically align label differently for
                                        # positive and negative values.


# Call the function above. All the magic happens there.
######add_value_labels(ax)
# Generate a bar plot showing the total number of mice for each treatment throughout the course of the study using pyplot.

##x_axis
drug_regimen_len=len(clean_study_data_complete["Drug Regimen"].unique())
drug_regimen_items=clean_study_data_complete["Drug Regimen"].unique()
drug_regimen_items_ticks = np.arange(len(drug_regimen_items))

x_axis = drug_regimen_items

#y_axis
mouse_id_len = clean_study_data_complete["Drug Regimen"].value_counts()
y_axis = mouse_id_len

#labels
plt.title("Drug Regimen Data Points")
plt.xlabel("Drug Regimen")
plt.ylabel("Number of Data Points")

#limits
plt.xlim(-0.75,len(x_axis)-0.25)
plt.ylim(0,max(y_axis)+20)

#tick locations
tick_locations = [value for value in drug_regimen_items_ticks]
plt.xticks(tick_locations, x_axis, rotation="vertical")

plt.bar(x_axis, y_axis)
plt.show()
# Generate a pie plot showing the distribution of female versus male mice using pandas
gender_count = clean_study_data_complete["Sex"].value_counts()
labels = gender_count.index

explode = (0.1,0)
colors = ["thistle","lightsteelblue"]

gender_pie = gender_count.plot(kind="pie", y=labels, title=("Gender Pie"), explode=explode, colors=colors, autopct="%1.2f%%", shadow=True, startangle=90)
gender_pie.set_ylabel("Gender")

plt.tight_layout()
plt.axis("equal")
plt.show()
# Generate a pie plot showing the distribution of female versus male mice using pyplot
gender_count = clean_study_data_complete["Sex"].value_counts()
labels = gender_count.index
explode = (0.1,0)
colors = ["thistle","lightsteelblue"]
plt.title("Gender Pie")
plt.ylabel("Gender")

plt.pie(gender_count, explode=explode, labels=labels, colors=colors, autopct="%1.2f%%", shadow=True, startangle=90)

plt.tight_layout()
plt.axis("equal")
plt.show()<jupyter_output><empty_output><jupyter_text>Quartiles, Outliers and Boxplots<jupyter_code># Calculate the final tumor volume of each mouse across four of the treatment regimens:  
# Capomulin, Ramicane, Infubinol, and Ceftamin

# Start by getting the last (greatest) timepoint for each mouse
rxfour = clean_study_data_complete[clean_study_data_complete["Drug Regimen"].isin(["Capomulin", "Ramicane", "Infubinol", "Ceftamin"])]

tplast = rxfour.groupby(["Drug Regimen","Mouse ID"])["Timepoint"].last().reset_index(name="Timepoint")

# Merge this group df with the original dataframe to get the tumor volume at the last timepoint
rx_merge = pd.merge(tplast, clean_study_data_complete, how="left", on="Mouse ID").rename(columns={"Drug Regimen_x":"Drug Regimen Last","Drug Regimen_y":"Drug Regimen","Timepoint_x":"Timepoint Last","Timepoint_y":"Timepoint"})

last = rx_merge["Timepoint"] == rx_merge["Timepoint Last"]

rx_merge_last = rx_merge[last]
rx_merge_last
# Put treatments into a list for for loop (and later for plot labels)
rx_drug = rx_merge_last.groupby(["Drug Regimen"]).nunique()
rx_list = rx_drug.index

# Create empty list to fill with tumor vol data (for plotting)
tumor_plot = []

# Calculate the IQR and quantitatively determine if there are any potential outliers. 
quartiles = rx_merge_last["Tumor Volume (mm3)"].quantile([.25,.5,.75])
lowerq = quartiles[0.25]
upperq = quartiles[0.75]
iqr = upperq-lowerq

print(f"The lower quartile of Tumor Volume (mm3) is: {lowerq}")
print(f"The upper quartile of Tumor Volume (mm3) is: {upperq}")
print(f"The interquartile range of Tumor Volume (mm3) is: {iqr}")
print(f"The the median of Tumor Volume (mm3) is: {quartiles[0.5]} ")

lower_bound = lowerq - (1.5*iqr)
upper_bound = upperq + (1.5*iqr)
print(f"Values below {lower_bound} could be outliers.")
print(f"Values above {upper_bound} could be outliers.")

outlier_tumor_volume = rx_merge_last.loc[(rx_merge_last["Tumor Volume (mm3)"] < lower_bound) | (rx_merge_last["Tumor Volume (mm3)"] > upper_bound)]
#outlier_tumor_volume
    
    # Locate the rows which contain mice on each drug and get the tumor volumes
rx_capomulin = rx_merge_last.loc[rx_merge_last["Drug Regimen"] == "Capomulin","Tumor Volume (mm3)"].values
rx_ramicane = rx_merge_last.loc[rx_merge_last["Drug Regimen"] == "Ramicane","Tumor Volume (mm3)"].values
rx_infubinol = rx_merge_last.loc[rx_merge_last["Drug Regimen"] == "Infubinol","Tumor Volume (mm3)"].values
rx_ceftamin = rx_merge_last.loc[rx_merge_last["Drug Regimen"] == "Ceftamin","Tumor Volume (mm3)"].values

    # add subset 
#rx_merge_boxplot = rx_merge_last_capomulin
#rx_merge_boxplot.append(rx_merge_last_ramicane)
#rx_merge_boxplot.append(rx_merge_last_infubinol)
#rx_merge_boxplot.append(rx_merge_last_ceftamin)
#rx_merge_boxplot
    
    # Determine outliers using upper and lower bounds<jupyter_output>The lower quartile of Tumor Volume (mm3) is: 37.187743802499995
The upper quartile of Tumor Volume (mm3) is: 59.930261755000004
The interquartile range of Tumor Volume (mm3) is: 22.74251795250001
The the median of Tumor Volume (mm3) is: 45.0 
Values below 3.0739668737499812 could be outliers.
Values above 94.04403868375002 could be outliers.<jupyter_text>Line and Scatter Plots<jupyter_code># Generate a line plot of time point versus tumor volume for a mouse treated with Capomulin

##capomulin dataset
rx_capomulin_trend = clean_study_data_complete[clean_study_data_complete["Drug Regimen"].isin(["Capomulin"])]

##x_axis
tp = rx_capomulin_trend.groupby(["Timepoint"]).nunique()
x_axis = tp.index

##y_axis
vol = rx_capomulin_trend.groupby(["Timepoint"]).sum()
y_axis = vol["Tumor Volume (mm3)"]

##plot
capomulin, = plt.plot(x_axis, y_axis, color="maroon", Label="Capomulin")

plt.title(" Capomulin Tumor Volume (mm3) by Timepoint")
plt.xlabel("Timepoint")
plt.ylabel("Average Tumor Volume (mm3)")

ymin = min(vol["Tumor Volume (mm3)"])-min(vol["Tumor Volume (mm3)"])*.05
ymax = max(vol["Tumor Volume (mm3)"])+max(vol["Tumor Volume (mm3)"])*.05
plt.ylim(ymin, ymax)

plt.show()
#----------------------------
#plt.scatter(x_axis, y_axis, marker="^", facecolors="maroon", edgecolors="black")

##https://stackoverflow.com/questions/30313882/scatterplot-with-different-size-marker-and-color-from-pandas-dataframe
##markers = ['^', 'o', 'v', 's', 'd']
# Generate a box plot of the final tumor volume of each mouse across four regimens of interest
red_black_marker = dict(markerfacecolor="r", marker="o")

plt.boxplot([rx_capomulin, rx_ramicane, rx_infubinol, rx_ceftamin], flierprops=red_black_marker, labels=["Capomulin","Ramicane","Ifubinol","Ceftamin"])

plt.title("Boxplot Tumor Volume (mm3) by Drug Regimen")
plt.xlabel("Drug Regimen")
plt.ylabel("Tumor Volume (mm3)")

plt.show()

#sym="r", notch=True
#https://matplotlib.org/gallery/pyplots/boxplot_demo_pyplot.html#sphx-glr-gallery-pyplots-boxplot-demo-pyplot-py
# Generate a scatter plot of mouse weight versus average tumor volume for the Capomulin regimen

rx_capomulin = clean_study_data_complete[clean_study_data_complete["Drug Regimen"].isin(["Capomulin"])]

mouse_id_key = rx_capomulin.groupby("Mouse ID").agg(mouse_weight=("Weight (g)","mean"),tumor_size=("Tumor Volume (mm3)","mean"))

x_axis = mouse_id_key["mouse_weight"]
y_axis = mouse_id_key["tumor_size"]

capomulin = plt.scatter(x_axis, y_axis, marker="d", facecolors="aqua", edgecolors="black")

plt.title("Capomulin Mouse Weight vs Avg Tumor Volume (mm3)")
plt.xlabel("Mouse Weight (g)")
plt.ylabel("Average Tumor Volume (mm3)")

xmin = min(mouse_id_key["mouse_weight"])-min(mouse_id_key["mouse_weight"])*.05
xmax = max(mouse_id_key["mouse_weight"])+max(mouse_id_key["mouse_weight"])*.05
plt.xlim(xmin, xmax)

ymin = min(mouse_id_key["tumor_size"])-min(mouse_id_key["tumor_size"])*.05
ymax = max(mouse_id_key["tumor_size"])+max(mouse_id_key["tumor_size"])*.05
plt.ylim(ymin, ymax)

plt.show()

##https://stackoverflow.com/questions/30313882/scatterplot-with-different-size-marker-and-color-from-pandas-dataframe
##markers = ['^', 'o', 'v', 's', 'd']
#----------------------------------
#https://stackoverflow.com/questions/60864585/groupby-sum-and-average-in-pandas-and-make-data-frame
    #df.groupby('ID').agg(['mean','sum'])
    #------------------------------
    #df1 = (df.groupby('ID').agg(Average_score=('Score','mean'), 
    #                            Total_score=('Score','sum'))
    #         .reset_index())
rx_mouse = clean_study_data_complete[clean_study_data_complete["Mouse ID"].isin(["l509"])]
#rx_mouse

#mouse_id_key = rx_mouse.groupby("Mouse ID").agg(mouse_weight=("Weight (g)","mean"),tumor_size=("Tumor Volume (mm3)","mean"))

x_axis = rx_mouse["Timepoint"]
y_axis = rx_mouse["Tumor Volume (mm3)"]

mouse_b128, = plt.plot(x_axis, y_axis, marker="d", color="red", label="L509")

plt.title("Capomulin Treatment for Mouse L509")
plt.xlabel("Timepoint")
plt.ylabel("Tumor Volume (mm3)")

xmin = min(rx_mouse["Timepoint"])-5
xmax = max(rx_mouse["Timepoint"])+5
plt.xlim(xmin, xmax)

ymin = min(rx_mouse["Tumor Volume (mm3)"])-min(rx_mouse["Tumor Volume (mm3)"])*.05
ymax = max(rx_mouse["Tumor Volume (mm3)"])+max(rx_mouse["Tumor Volume (mm3)"])*.05
plt.ylim(ymin, ymax)

plt.show()
#capomulin, = plt.plot(x_axis, y_axis, color="maroon", Label="Capomulin")<jupyter_output><empty_output><jupyter_text>Correlation and Regression<jupyter_code># Calculate the correlation coefficient and linear regression model 
# for mouse weight and average tumor volume for the Capomulin regimen
rx_capomulin = clean_study_data_complete[clean_study_data_complete["Drug Regimen"].isin(["Capomulin"])]

mouse_id_key = rx_capomulin.groupby("Mouse ID").agg(mouse_weight=("Weight (g)","mean"),tumor_size=("Tumor Volume (mm3)","mean"))

x_axis = mouse_id_key["mouse_weight"]
y_axis = mouse_id_key["tumor_size"]

x_values = x_axis.values
y_values = y_axis.values

(slope, intercept, rvalue, pvalue, stderr) = linregress(x_values, y_values)
regress_values = x_values * slope + intercept
line_eq = "y = "  + str(round(slope,2)) + "x + " + str(round(intercept,2))
plt.scatter(x_values, y_values)

print(f"The slope is {str([slope])}, y-intercept {str([intercept])}, and correlation is {str([rvalue])}")

plt.plot(x_values,regress_values,"r-")
plt.annotate(line_eq,(6,10),fontsize=15,color="red")

plt.title("Capomulin Tumor Volume (mm3) by Weight (g)")
plt.xlabel("Weight (g)")
plt.ylabel("Tumor Volume (mm3)")
plt.show()

#https://stackoverflow.com/questions/36420908/can-i-draw-a-regression-line-and-show-parameters-using-scatterplot-with-a-pandas
#import seaborn as sns
#sns.lmplot(x='one',y='two',data=df,fit_reg=True)<jupyter_output>The slope is [0.9544396890241045], y-intercept [21.552160532685015], and correlation is [0.8419363424694718]
{'fraction_non_alphanumeric': 0.1346669919492559, 'fraction_numerical': 0.0211027079775555, 'mean_word_length': 5.187547169811321, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 24, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '98154', 'license': 'ADSL', 'n_tokens_mistral': 5962, 'n_tokens_neox': 5490, 'n_words': 1369, 'path': 'pymaceuticals/pymaceuticals.ipynb', 'repo_name': 'cliffeandrew/pymaceuticals-challenge', 'size': '16396'}