<jupyter_start><jupyter_text>Healthsites.io bulk data import This notebook imports a bulk data export from [Healthsites.io](healthsites.io).<jupyter_code>import pandas as pd
import csv
# papermill parameters
output_folder = "../output/"
cols_to_keep = ["HealthX", "HealthY", "healthcare", "name", "operator", "beds", "staff_doct", "staff_nurs"]

df = pd.read_csv("https://s3-us-west-1.amazonaws.com/starschema.covid/Healthsites_io_World_2020-03-19.csv", 
                 dtype={'beds': 'Int32', 'name': str, 'operator': str},
                 usecols=["HealthX", "HealthY", "healthcare", "name", "operator", "beds", "staff_doct", "staff_nurs"],
                 encoding="ISO-8859-1",
                 keep_default_na=False)
df["staff_doct"] = pd.to_numeric(df.staff_doct, errors='coerce')
df["staff_nurs"] = pd.to_numeric(df.staff_nurs, errors='coerce')<jupyter_output><empty_output><jupyter_text>Filter to columns and rename columnsWe are only interested in `lat`, `long`, type (`healthcare` -> `healthcare_provider_type`), `name`, `operator`, bed count (`beds`), staffing (`staff_doct` -> `staff_medical` and `staff_nurs` -> `staff_nursing`).We are also excluding * indirect medical specialities (e.g. `diagnostics`), * chronic care (e.g. dialysis), * mental health provision incl. drug rehabilitation, * labour and delivery,* specialities that do not play a role in COVID-19 response (dentistry, optometry, physiotherapy &c.), and * alternative medicine and paramedicine.Finally, we map various local names and different spellings into the following categories:* `primary_care` (GPs, PCPs, paediatrics),* `secondary_care`,* `hospital`s incl. polyclinics,* `emergency_and_urgent_care`,* `health_center`s, incl. multispeciality care.<jupyter_code>cols_name_mapping = {"HealthX": "Long",
                     "HealthY": "Lat",
                     "healthcare": "Healthcare_Provider_Type",
                     "name": "Name",
                     "operator": "Operator",
                     "beds": "Beds",
                     "staff_doct": "Staff_Medical",
                     "staff_nurs": "Staff_Nursing"}
healthcare_provision_mapping = {"hospital": "hospital",
                                "pharmacy": "pharmacy",
                                "centre": "health_center",
                                "clinic": "secondary_care",
                                "doctor": "primary_care",
                                "yes": "primary_care",
                                "general_practitioner": "primary_care",
                                "several_doctors": "primary_care",
                                "urgent_care": "emergency_and_urgent_care",
                                "health_center": "health_center",
                                "Polyclinic": "hospital",
                                "Health Centre": "health_center",
                                "centre_de_santé": "health_center",
                                "general_medicine": "primary_care",
                                "Hospital": "hospital",
                                "group": "health_center",
                                "doctors": "health_center",
                                "ambulatorio": "health_center",
                                "primary hospital": "hospital",
                                "pediatrics": "primary_care",
                                "general;paediatrics": "primary_care",
                                "doctor; pharmacy": "primary_care",
                                "polyclinic": "hospital",
                                "heath center": "health_center",
                                "paramedic": "emergency_and_urgent_care",
                                "doctor;midwife": "primary_care",
                                "hospital;pharmacy": "hospital",
                                "family_practice": "primary_care",
                                "nurse": "primary_care",
                                "neurologist": "secondary_care",
                                "doctor;dentist;physiotherapist;psychotherapist": "health_center",
                                "geriatric": "primary_care",
                                "laboratory;doctor": "health_center",
                                "surgery": "primary_care",
                                "doctor;dentist;physiotherapist": "health_center",
                                "internist": "primary_care",
                                "physician_assistant": "emergency_and_urgent_care",
                                "general": "primary_care",
                                "doctor;nurse": "primary_care",
                                "generalist": "primary_care",
                                "clinik": "hospital",
                                "clinic;dentist;doctor;pharmacy": "health_center",
                                "pharmacy;laboratory;doctor": "health_center",
                                "nursing": "primary_care",
                                "pediatrician": "primary_care",
                                "General Physician": "primary_care",
                                "centre;doctor": "health_center",
                                "polyambulatory": "health_center",
                                "paediatrics": "primary_care",
                                "clinic;doctors;dialysis": "health_center",
                                "dispensary": "pharmacy",
                                "doctor;hospital": "hospital",
                                "center": "health_center",
                                "otolaryngologist": "secondary_care",
                                "first_aid": "emergency_and_urgent_care",
                                "clinic;laboratory;physiotherapist;occupational_therapist": "health_center",
                                "doctor;pharmacy": "primary_care",
                                "pharmacy;doctor": "primary_care",
                                "doctor;pharmacy;hospital;dentist;laboratory;birthing_center": "health_center",
                                "doctor;pharmacy;dentist;laboratory": "health_center",
                                "doctor;pharmacy;hospital;dentist;physiotherapist;alternative;laboratory;optometrist;birthing_center": "health_center",
                                "clinic;doctor": "health_center",
                                "Calindagan Health Center": "health_center",
                                "doctor+pharmacy": "health_center",
                                "health_post": "primary_care",
                                "emergency_ward": "emergency_and_urgent_care",
                                "laboratory, pharmacy, clinic": "health_center",
                                "dentist;doctors": "health_center",
                                "Rehaklinik für Orthopädie und rheumatische Erkrankungen": "secondary_care",
                                "nephrology": "secondary_care",
                                "*": ""}
df.rename(columns=cols_name_mapping, inplace=True)
df["Healthcare_Provider_Type"] = df["Healthcare_Provider_Type"].replace(healthcare_provision_mapping)
df = df[df["Healthcare_Provider_Type"].isin(healthcare_provision_mapping.values())]
df.to_csv(output_folder + "HS_BULK_DATA.csv", index=False, quoting=csv.QUOTE_NONNUMERIC)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1117575429576512, 'fraction_numerical': 0.002570694087403599, 'mean_word_length': 1.5900490539593553, 'pattern_counts': {'":': 79, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '339261', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2034, 'n_tokens_neox': 1824, 'n_words': 326, 'path': 'notebooks/HS_BULK_DATA.ipynb', 'repo_name': 'korompaiistvan/COVID-19-data', 'size': '7391'}