<jupyter_start><jupyter_text>Clean up `gender` column:- all similiar to male as male- all similiar to female as female- all others as other- only 3 'nulls', for simplicity to other<jupyter_code>clean_data['gender'].unique()
clean_data.loc[clean_data['gender'].str.contains(r'(^\s*[Ff]emale)', na=False),'gender'] = "Female"
clean_data.loc[clean_data['gender'].str.contains(r'(^[Ww]oman)', na=False),'gender'] = "Female"
clean_data.loc[clean_data['gender'].str.contains(r'(^[Ff]$)', na=False),'gender'] = "Female"
clean_data.loc[clean_data['gender'].str.contains(r'(^[Mm]ale)', na=False),'gender'] = "Male"
clean_data.loc[clean_data['gender'].str.contains(r'(^[Mm]ALE)', na=False),'gender'] = "Male"
clean_data.loc[clean_data['gender'].str.contains(r'(^[Mm]$)', na=False),'gender'] = "Male"
clean_data.loc[clean_data['gender'].str.contains(r'(^[Mm]an)', na=False),'gender'] = "Male"
clean_data.loc[clean_data['gender'].str.contains(r'(^[Mm]ail)', na=False),'gender'] = "Male"
clean_data.loc[clean_data['gender'].str.contains(r'(^nan)', na=False),'gender'] = np.NaN
clean_data.loc[clean_data['gender'].str.contains('M\|', na=False),'gender'] = "Male"


#all remaining as `Other`
clean_data.loc[clean_data['gender'].str.contains(r'[^Male|Female]', na=False),'gender'] = "Other"
clean_data.loc[clean_data['gender'].isnull(),'gender'] = "Other"
#check output contains 3 possible values
clean_data['gender'].unique()

clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'(^\s*[Ff]emale)', na=False),'What is your gender?'] = "Female"
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'(^[Ww]oman)', na=False),'What is your gender?'] = "Female"
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'(^[Ff]$)', na=False),'What is your gender?'] = "Female"
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'(^[Mm]ale)', na=False),'What is your gender?'] = "Male"
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'(^[Mm]ALE)', na=False),'What is your gender?'] = "Male"
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'(^[Mm]$)', na=False),'What is your gender?'] = "Male"
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'(^[Mm]an)', na=False),'What is your gender?'] = "Male"
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'(^[Mm]ail)', na=False),'What is your gender?'] = "Male"
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'(^nan)', na=False),'What is your gender?'] = np.NaN
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains('M\|', na=False),'What is your gender?'] = "Male"

#all remaining as `Other`
clean_data_questions.loc[clean_data_questions['What is your gender?'].str.contains(r'[^Male|Female]', na=False),'What is your gender?'] = "Other"
clean_data_questions.loc[clean_data_questions['What is your gender?'].isnull(),'What is your gender?'] = "Other"
#check output contains 3 possible values
clean_data_questions['What is your gender?'].unique()<jupyter_output><empty_output><jupyter_text>Re-word response category in `mental_health_benefits_healthcare` question to shorten:<jupyter_code>clean_data.loc[clean_data['mental_health_benefits_healthcare'].str.contains('Not eligible for coverage / N/A', na=False),'mental_health_benefits_healthcare'] = "Not eligible for coverage"
clean_data_questions.loc[clean_data_questions['Are mental health benefits provided as part of healthcare?'].str.contains('Not eligible for coverage / N/A', na=False),
                         'Are mental health benefits provided as part of healthcare?'] = "Not eligible for coverage"
#Check:
clean_data['mental_health_benefits_healthcare'].unique()
clean_data.to_csv("../data/processed/mental_health_clean.csv", index=0)
clean_data_questions.to_csv("../data/processed/mental_health_clean2.csv", index=0)
data_check2 = pd.read_csv("../data/processed/mental_health_clean2.csv")
data_check = pd.read_csv("../data/processed/mental_health_clean.csv")
data_check.shape
data_check2.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18711656441717792, 'fraction_numerical': 0.0021236432279377066, 'mean_word_length': 8.789838337182449, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '383500', 'license': 'MIT', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1408, 'n_words': 249, 'path': 'src/Processing.ipynb', 'repo_name': 'UBC-MDS/group-19', 'size': '4238'}