<jupyter_start><jupyter_text>Matplotlib Challenge - Observations and Insights  Grader the written observations are at the end of this file<jupyter_code>%matplotlib inline 

# Dependencies and Setup
import matplotlib.pyplot as plt
import pandas as pd
import scipy.stats as st
import numpy as np

# Study data files
mouse_metadata_path = "../data/Mouse_metadata.csv"
study_results_path = "../data/Study_results.csv"

# Read the mouse data and the study results
mouse_metadata = pd.read_csv(mouse_metadata_path)
study_results = pd.read_csv(study_results_path)
study_results.head()
# Before beginning the analysis, check the data for any mouse ID with duplicate time points and remove any data associated with that mouse ID.

# Combine the data into a single dataset
mice_data = mouse_metadata.merge(study_results, on='Mouse ID', how='outer')
# Display the data table for preview
mice_data.head()
# Functions that will be used through out the program

# Function to gather data for a requested drug regimen and return a dataframe
# of the max timepoints for each mouse in the given regimen.
def get_drug_data(drug_name):
    # TODO Error check to make sure drug name passed is in the data
    drug_data = sorted_mice_data.loc[sorted_mice_data["Drug Regimen"] == drug_name]
    drug_max_time = pd.DataFrame(drug_data.groupby('Mouse ID').max()['Timepoint'])
    drug_merge = drug_max_time.merge(clean_mice_data, on=("Mouse ID", "Timepoint"), how="left")
    return drug_merge

# Function to return data for the box whisker plot
def get_drug_tumor_data(drug_name):
    tumor_vol = get_drug_data(drug_name)['Tumor Volume (mm3)']
    return tumor_vol

# Function to return a string of all statistical data for a given regimen.
def get_drug_stats(drug_name):
    tumor_vol = get_drug_tumor_data(drug_name)
    quartiles = tumor_vol.quantile([0.25,0.5,0.75])
    drug_lowerq = quartiles[0.25]
    drug_upperq = quartiles[0.75]
    drug_iqr = drug_upperq - drug_lowerq
    drug_lower_bound = drug_lowerq - 1.5 * drug_iqr
    drug_upper_bound = drug_upperq + 1.5 * drug_iqr    
    return_string = "-------------------------\n"
    return_string += f"Statistical data for {drug_name}\n"
    return_string += f"\tIQR = {drug_iqr:.2f}\n"
    return_string += f"\tLower Quartile = {drug_lowerq:.2f}\n"
    return_string += f"\tUpper Quartile = {drug_upperq:.2f}\n"
    return_string += f"\tLower Bound: {drug_lower_bound:.2f}\n"
    return_string += f"\tUpper Bound: {drug_upper_bound:.2f}\n"
    return_string += f"\t{drug_name} potential outliers could be values below {drug_lower_bound:.2f} and above {drug_upper_bound:.2f} \n"
    return_string += "-------------------------\n"
    return return_string
# Checking the number of mice, total rows, and duplicate data.
tot_cnt = len(mice_data["Mouse ID"].unique())
print(f"Total number of mice: {tot_cnt}")

tot_rows = mice_data.shape[0]
print(f"Total number of rows: {tot_rows}")

tot_dups = mice_data.duplicated().sum()
print(f"Total number of duplicate mice: {tot_dups}")
# Getting the duplicate mice by ID number that shows up for Mouse ID and Timepoint. 
dup_mice = study_results.loc[study_results.duplicated(subset=['Mouse ID', 'Timepoint',]), 'Mouse ID'].unique()
for dup_mouse in dup_mice:
    print(f"Duplicated mice ID: {dup_mouse}")

# Optional: Get all the data for the duplicate mouse ID. 
dup_data_list = []
for dup_mouse in dup_mice:
    dup_data_list.append(mice_data[mice_data['Mouse ID'].str.contains(dup_mouse)])

dup_data_df = pd.DataFrame(dup_data_list[0], columns=list(mice_data))
dup_data_df
# Create a clean DataFrame by dropping the duplicate mouse by its ID.
for dup_mouse in dup_mice:
    clean_mice_data = mice_data[mice_data["Mouse ID"] != dup_mice[0]]
# Checking the number of mice in the clean DataFrame.
cleaned_mice_cnt = len(clean_mice_data["Mouse ID"].unique())
cleaned_mice_row_cnt = clean_mice_data.shape[0]
print(f"Number of mice in the cleaned dataframe: {cleaned_mice_cnt}")
print(f"Number of rows in the cleaned dataframe: {cleaned_mice_row_cnt}")
# Show the cleaned data frame
clean_mice_data.to_html('../../pages/data_html.html', classes=["table-bordered", "table-striped", "table-hover"], index=False, justify="left")
clean_mice_data<jupyter_output><empty_output><jupyter_text>Summary Statistics<jupyter_code># Generate a summary statistics table of mean, median, variance, standard deviation, and SEM of the tumor volume for each regimen
# Use groupby and summary statistical methods to calculate the following properties of each drug regimen: 
# mean, median, variance, standard deviation, and SEM of the tumor volume. 
mean = clean_mice_data.groupby('Drug Regimen')['Tumor Volume (mm3)'].mean()
median = clean_mice_data.groupby('Drug Regimen')['Tumor Volume (mm3)'].median()
variance = clean_mice_data.groupby('Drug Regimen')['Tumor Volume (mm3)'].var()
std = clean_mice_data.groupby('Drug Regimen')['Tumor Volume (mm3)'].std()
sem = clean_mice_data.groupby('Drug Regimen')['Tumor Volume (mm3)'].sem()

# Assemble into a single summary dataframe.
summary_stats = pd.DataFrame({"Mean": mean, "Median": median, "Variance": variance, "Standard Deviation": std, "SEM": sem})
summary_stats_indexed = summary_stats.reset_index()
summary_stats_indexed.to_html('../../pages/data_stats_html.html', classes=["table-bordered", "table-striped", "table-hover"], index=False, justify="left")
summary_stats

# Generate a summary statistics table of mean, median, variance, standard deviation, and SEM of the tumor volume for each regimen
# Using the aggregation method, produce the same summary statistics in a single line
mice_stats_agg = clean_mice_data.groupby('Drug Regimen').agg({'Tumor Volume (mm3)' : ['mean', 'median', 'var', 'std','sem']})
mice_stats_agg<jupyter_output><empty_output><jupyter_text>Bar and Pie Charts<jupyter_code># Generate the data for the number of measurements for each drug regimen 
# and sort ascending by number of measurements
drug_data = clean_mice_data.groupby('Drug Regimen')['Mouse ID'].count().sort_values(ascending=False)
drug_data_1 = clean_mice_data.groupby('Drug Regimen', as_index = False)['Mouse ID'].count().sort_values(by='Mouse ID',ascending=False)
drug_data_1.rename(columns = {'Mouse ID':'Measurments'}, inplace = True)
drug_data_1.to_html('../../pages/drug_data_html.html', classes=["table-bordered", "table-striped", "table-hover"], index=False, justify="left")
drug_data
# Generate a bar plot showing the total number of measurements taken on each drug regimen using pandas.
pandas_drug_plot = drug_data.plot(kind='bar',title='Measurements for each Drug Regimen')
pandas_drug_plot.set_ylabel('Number of Measurements')
plt.show()
# Generate a bar plot showing the total number of measurements taken on each drug regimen using pyplot.
plt.bar(drug_data.index, drug_data.values)
plt.ylabel('Number of Measurements')
plt.title('Measurements for each Drug Regimen')
plt.xticks(rotation=90)
plt.tight_layout()
fig = plt.gcf()
fig.savefig('../../images/measurements_regimen.png')
plt.show()
# Generate the data for the distibution by sex
# TODO Make output nice
sex_data = clean_mice_data.groupby('Sex')['Mouse ID'].nunique()
sex_df = sex_data.to_frame()
sex_df
# Generate a pie plot showing the distribution of female versus male mice using pandas
# TODO Make Title size 15
colors = ['lightpink', 'lightblue']
pandas_sex_plot = sex_data.plot(kind='pie',title='Mice by Sex',autopct='%1.1f%%', startangle=270, fontsize=15, colors = colors)
pandas_sex_plot.set_ylabel('')
plt.show()
# Generate a pie plot showing the distribution of female versus male mice using pyplot
plt.rcParams['font.size'] = 15
plt.pie(sex_data.values, labels=sex_data.index, autopct="%1.1f%%", shadow=False, startangle=270, colors=colors)
plt.axis("equal")
plt.title('Mice by Sex')
plt.tight_layout()
fig = plt.gcf()
fig.savefig('../../images/mice_gender.png')
plt.show()<jupyter_output><empty_output><jupyter_text>Quartiles, Outliers and Boxplots<jupyter_code># Calculate the final tumor volume of each mouse across four of the treatment regimens:  
# Capomulin, Ramicane, Infubinol, and Ceftamin
drug_list = ["Capomulin","Ramicane","Infubinol","Ceftamin",]

sorted_mice_data = clean_mice_data.sort_values(["Timepoint"], ascending=True)

# Tumor volumes at last timepoint for every drug
regimen_data = sorted_mice_data[["Drug Regimen", "Mouse ID", "Timepoint", "Tumor Volume (mm3)"]]
regimen_data
# Show statistical data for the 4 drugs requested
for drug in drug_list:
    print(get_drug_stats(drug))
# Generate a box plot of the final tumor volume of each mouse across four regimens of interest
treatment_plot = []
for drug in drug_list:
    treatment_plot.append(get_drug_tumor_data(drug))
    
fig1, ax1 = plt.subplots()
ax1.set_title('Final Tumor Volume by Regimen')
ax1.set_ylabel('Final Tumor Volume (mm3)')
ax1.set_xlabel('Drug Regimen')

ax1.boxplot(treatment_plot, labels=drug_list)
fig = plt.gcf()
fig.savefig('../../images/tumor_vol_regimen.png')
plt.show()<jupyter_output><empty_output><jupyter_text>Line and Scatter Plots<jupyter_code># Generate a line plot of tumor volume vs. time point for a random mouse treated with Capomulin
# Using a seed of 42 for reporducability ... DON'T PANIC
drug_regimen = "Capomulin"
mouse_id = get_drug_data(drug_regimen).sample(random_state=42)["Mouse ID"].iloc[0]
line_plot_data = clean_mice_data.loc[(clean_mice_data["Mouse ID"] == mouse_id)]
line_plot_data = line_plot_data.set_index("Timepoint")

#Final plot
line_plot_data["Tumor Volume (mm3)"].plot(color = "blue")
plt.title(f"Tumor Volume of Mouse {mouse_id} Over Time")
plt.xlabel("Timepoint")
plt.ylabel("Tumor Volume (mm3)")
plt.tight_layout()

fig = plt.gcf()
fig.savefig('../../images/overtime_regimen.png')
plt.show()
# Generate a scatter plot of average tumor volume vs. mouse weight for the Capomulin regimen
scatter_plot_data = clean_mice_data.loc[(clean_mice_data["Drug Regimen"] == drug_regimen)]
scatter_plot_data = scatter_plot_data.groupby(["Mouse ID"]).mean()

# set x and y value
weight_data = scatter_plot_data["Weight (g)"]
volume_data = scatter_plot_data["Tumor Volume (mm3)"]

# Plot the graph
plt.scatter(weight_data, volume_data, color = "blue")
plt.xlabel("Weight (g)")
plt.ylabel("Average Tumor Volume (mm3)")
plt.title(f"Weight Versus Average Tumor Volume for {drug_regimen}")
plt.tight_layout()

fig = plt.gcf()
fig.savefig('../../images/wgt_vol.png')
plt.show()<jupyter_output><empty_output><jupyter_text>Correlation and Regression<jupyter_code># Calculate the correlation coefficient and linear regression model 
# for mouse weight and average tumor volume for the Capomulin regimen
corr_coeff = st.pearsonr(weight_data, volume_data)[0]
print(f'The correlation coefficient between weight and tumor volume is {corr_coeff:.2f}')

# Linear regression model
slope, intercept, r_value, p_value, std_err = st.linregress(weight_data, volume_data)

print(f"Slope: {slope}")
print(f"Intercept: {intercept}")
print(f"R Value: {r_value}")
print(f"P value: {p_value}")
print(f"Standard Error: {std_err}")     
#using prior computing data to plot the regression line on the scatter plot
plt.scatter(weight_data, volume_data, color = "blue")
plt.plot(weight_data, slope * weight_data + intercept, color = "red")
plt.xlabel("Weight (g)")
plt.ylabel("Average Tumor Volume (mm3)")
plt.title(f"Linear Regression on scatter plot for {drug_regimen}")
plt.tight_layout()

fig = plt.gcf()
fig.savefig('../../images/linreg_capomulin.png')
plt.show()<jupyter_output><empty_output><jupyter_text>Observations and InferencesLook across all previously generated figures and tables and write at least three observations or inferences that can be made from the data. Include these observations at the top of notebook.1. Out of all the regimens of interest Capomulin and Ramicane showed the best decrease of tumor volume over the course.2. With Capomulin there is a decrease of weight with the decrease of tumor volume.3. Infubinol was the only regimen with a significant outlier.4. The two regimens that were most effective also had the most data points (mice) in the study.<jupyter_code># For giggles wanted to see the box wisker plot for all drugs
treatment_plot = []
drug_list_tot = clean_mice_data["Drug Regimen"].unique()
for drug in drug_list_tot:
    treatment_plot.append(get_drug_tumor_data(drug))
    
fig1, ax1 = plt.subplots()
ax1.set_title('Final Tumor Volume by Regimen')
ax1.set_ylabel('Final Tumor Volume (mm3)')
ax1.set_xlabel('Drug Regimen')

ax1.boxplot(treatment_plot, labels=drug_list_tot)
plt.xticks(rotation=90)
plt.tight_layout()

fig = plt.gcf()
fig.savefig('../../images/final_vol_regimen.png')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1202561669829222, 'fraction_numerical': 0.007194813409234661, 'mean_word_length': 5.837297297297297, 'pattern_counts': {'":': 5, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '163852', 'license': 'MIT', 'n_tokens_mistral': 4374, 'n_tokens_neox': 4118, 'n_words': 1184, 'path': 'pymaceuticals/resources/src/pymaceuticals.ipynb', 'repo_name': 'bgalde-dev/web-design-challenge', 'size': '12648'}