<jupyter_start><jupyter_text>Data Import  Line list2015-2016 (through June 2016) in stata format<jupyter_code>line_list_raw = (pd.read_stata(DATA_IN+'linelist_latin_20160717.dta', convert_dates=True)
                     .drop(['agemonth', 'ageyear'], axis=1)
                     .set_index('id'))<jupyter_output><empty_output><jupyter_text>Convert remaining date fields<jupyter_code>line_list_date_cols = ['onsetofdisease'] + line_list_raw.columns[line_list_raw.columns.str.contains('date')].tolist()

for col in line_list_date_cols:
    try:
        line_list_raw[col] = pd.to_datetime(line_list_raw[col])
    except ValueError:
        pass<jupyter_output><empty_output><jupyter_text>Drop fields with >75% missing values.<jupyter_code>line_list_raw = line_list_raw.dropna(axis=1, thresh=int(line_list_raw.shape[0]*0.10))
line_list_raw.shape
line_list_raw.head()
cols_to_keep = ['date_onset', 'provincecity', 'sex', 'dob_final', 'age_years', 'died_enhanced', 
                'confirmed', 'labconfirmed', 'date_lab', 'vaccine_ever']

line_list = line_list_raw[cols_to_keep]
line_list.head()<jupyter_output><empty_output><jupyter_text>Additional line list data (July to December 2016)<jupyter_code>line_list_2016_raw = pd.read_csv(DATA_IN + 'MNG2016 (July to Dec).csv', index_col=0, parse_dates=True)
line_list_2016_raw.columns = ['_'.join(x) for x in line_list_2016_raw.columns.str.lower().str.replace('\r','').str.split(' ').values]
for date_field in ('donset', 'dob', 'dscoll', 'dlabrec'):
    line_list_2016_raw[date_field] = pd.to_datetime(line_list_2016_raw[date_field])
cols_to_keep_2016 = ['donset', 'infection_province', 'sex', 'dob', 'age', 'outcome',  
                'mclass', 'mconfmeth', 'dlastvac', 'dlabrec']
line_list_2016 = line_list_2016_raw[cols_to_keep_2016].copy()<jupyter_output><empty_output><jupyter_text>Recode variables to match main dataset<jupyter_code>province_lookup = {'AR':'Arkhangai',
    'Bay':'Bayan Ulgii',
    'BKh':'Bayankhongor',
    'BU':'Bulgan',
    'DA':'Darkhan-Uul',
    'DG':'Dornogobi',
    'DO':'Dornod',
    'DU':'Dundgovi',
    'GA':'Gobi altai',
    'GS':'Govisumber',
    'KhE':'Khentii',
    'KhO':'Khovd',
    'KhU':'Khuvsgul',
    'OR':'Orkhon',
    'SB':'Sukhbaatar',
    'SE':'Selenge',
    'TU':'Tuv',
    'UB':'Ulaanbaatar',
    'UG':'Umnogovi',
    'UV':'Uvurkhangai',
    'UVs':'Uvs',
    'ZA':'Zavkhan'}

inverse_province_lookup = res = dict((v,k) for k,v in province_lookup.items())
line_list_2016 = (line_list_2016.assign(provincecity=line_list_2016.infection_province.replace(inverse_province_lookup),
                                       confirmed=(line_list_2016.mclass==1).astype(int),
                                       labconfirmed=((line_list_2016.mclass==1) 
                                                     & (line_list_2016.mconfmeth==1)).astype(int),
                                       male=(line_list_2016.sex=='Male').astype(int),
                                       died=(line_list_2016.outcome==2).astype(int),
                                       vaccine_ever=line_list_2016.dlastvac.notnull().astype(int))
                        .drop(['outcome', 'mconfmeth', 'mclass', 'sex', 'infection_province', 'dlastvac'], axis=1)
                        .rename(columns={'age':'age_years', 
                                         'donset':'date_onset',
                                        'dlabrec':'date_lab'})
                 )
line_list_2016.shape
line_list = (line_list.assign(male=line_list.sex=='Male')
                    .drop(['sex'], axis=1)
                    .rename(columns={'dob_final':'dob',
                                    'died_enhanced':'died'})
            )
line_list.shape
line_list_merged = pd.concat([line_list, line_list_2016], ignore_index=True)
line_list_merged.head()
line_list_merged.shape
line_list['month_onset'] = line_list.date_onset.dt.month
line_list['year_onset'] = line_list.date_onset.dt.year
line_list.isnull().mean().round(2)<jupyter_output><empty_output><jupyter_text>Monthly distribution of all cases<jupyter_code>line_list.groupby(['year_onset', 'month_onset']).date_onset.count().plot()
line_list.groupby(['year_onset', 'month_onset']).date_lab.apply(lambda x: x.notnull().mean()).plot()<jupyter_output><empty_output><jupyter_text>Monthly distribution of lab-confirmed only<jupyter_code>line_list[line_list.labconfirmed==1].groupby(['year_onset', 'month_onset']).date_onset.count().plot()
line_list[line_list.confirmed==1].groupby(['year_onset', 'month_onset']).date_onset.count().plot()<jupyter_output><empty_output><jupyter_text>Export merged data<jupyter_code>line_list.to_csv(DATA_OUT+'line_list.csv')<jupyter_output><empty_output><jupyter_text>Biweekly counts of cases<jupyter_code>line_list.head()<jupyter_output><empty_output><jupyter_text>Assign age groups<jupyter_code>age_classes = [0,1,5,10,15,20,25,30,35,40,100]
line_list['age_group'] = pd.cut(line_list.age_years, age_classes, right=False)
dates_index = line_list.groupby(['date_onset', 'age_group']).size().unstack().index
confirmed_data = line_list.query('confirmed==1')
confirmed_counts_2w = (confirmed_data
                        .groupby(['date_onset', 'age_group'])
                        .size()
                        .unstack()
                        .reindex(dates_index)
                        .fillna(0)
                        .resample('2W')
                        .sum())
confirmed_counts_2w.plot(cmap='viridis')
confirmed_counts_2w.to_csv(DATA_OUT+'confirmed_counts.csv')<jupyter_output><empty_output><jupyter_text>Under-5 subset<jupyter_code>confirmed_counts_U5 = (confirmed_data.dropna(subset=['age_years'])
                       .assign(age_years_int=confirmed_data.dropna(subset=['age_years']).age_years.astype(int))
                       .query('age_years<5')
                        .groupby(['date_onset', 'age_years_int'])
                        .size()
                        .unstack()
                        .reindex(dates_index)
                        .fillna(0)
                        .resample('2W')
                        .sum())
confirmed_counts_U5.plot(cmap='viridis')<jupyter_output><empty_output><jupyter_text>Birth cohortRoutine and SIA vaccination coverage, birth cohort population size, and total suspected cases in each year since 1925MCV is coverage proportion<jupyter_code>birth_cohort_raw = pd.read_csv(DATA_IN+'birth cohort.csv', index_col=0)
birth_cohort_raw.tail(10)
birth_cohort_raw.isnull().sum()
coverage_cols = (birth_cohort_raw.columns.str.startswith('SIA') |
                 birth_cohort_raw.columns.str.startswith('MCV'))
coverage = birth_cohort_raw[birth_cohort_raw.columns[coverage_cols]].fillna(0)
coverage.to_csv(DATA_OUT + 'coverage.csv')<jupyter_output><empty_output><jupyter_text>PopulationPopulation by age group in each province<jupyter_code>population = pd.read_csv(DATA_OUT+'denominators.csv', index_col=0)
population.head()<jupyter_output><empty_output><jupyter_text>Supplemental immunizationDetails of coverage of outbreak response campaigns in 2015 and 2016, by age group and Aimag<jupyter_code>sia_2015 = pd.read_excel(DATA_IN+'MR SIA 2015.xlsx', skiprows=49, index_col=0).dropna(axis=1)
sia_2015
sia_2015[list(range(5))].sum(1)
#sia_2015_summed = pd.concat([sia_2015[list(range(5))].sum(1), sia_2015[5]], axis=1)
#sia_2015_summed.columns = ['0-4', '5-9']
sia_2015_summed = pd.concat([sia_2015[0], sia_2015[list(range(1,5))].sum(1), sia_2015[5]], axis=1)
sia_2015_summed.columns = ['0', '1-4', '5-9']
sia_2015_summed.head()
sia_2016 = pd.read_excel(DATA_IN+'MR SIA 2016.xlsx', skiprows=2, index_col=0).drop('TOTAL').dropna()
sia_2016.index.name = 'age'
sia_2016.index = pd.Series(sia_2016.index.str.split(' ')).apply(lambda x: int(x[0]))
sia_2016_summed = pd.concat([sia_2016.loc[:19].sum(),
                               sia_2016.loc[20:24].sum(),
                               sia_2016.loc[25:29].sum(),
                               sia_2016.loc[30]], axis=1).drop('TOTAL').astype(int)
sia_2016_summed.columns = ['15-19', '20-24', '25-29', '30-34']
sia_2016_summed.index = sia_2015_summed.index
sia_2016_summed.head()
sia_2015_summed.to_csv(DATA_OUT + 'sia_2015.csv')
sia_2016_summed.to_csv(DATA_OUT + 'sia_2016.csv')<jupyter_output><empty_output><jupyter_text>Contact Matrix<jupyter_code>labels = ('0-4','5-9','10-14','15-19','20-24','25-29','30-34','35-39','40-44',
          '45-49','50-54','55-59','60-64','65-69','70-74','75-79')
contact_matrix = pd.read_excel(DATA_IN + 'MUestimates_all_locations_1.xlsx', 
                               sheet_name='Mongolia')
contact_matrix.index = labels
contact_matrix.columns = labels

contact_matrix
contact_matrix.to_csv(DATA_OUT + 'contact_matrix.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.153247050738747, 'fraction_numerical': 0.04821898980643683, 'mean_word_length': 3.753402286336418, 'pattern_counts': {'":': 0, '<': 62, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '61577', 'license': 'MIT', 'n_tokens_mistral': 3494, 'n_tokens_neox': 3081, 'n_words': 355, 'path': 'notebooks/Data Processing.ipynb', 'repo_name': 'fonnesbeck/mongolia_measles', 'size': '8731'}