<jupyter_start><jupyter_text>Exploratory Data AnalysisPreparing the BRFSS dataset<NAME>[MIT License](https://en.wikipedia.org/wiki/MIT_License)<jupyter_code>%matplotlib inline

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style='white')

import utils
from utils import decorate
from distribution import Pmf, Cdf<jupyter_output><empty_output><jupyter_text>Loading and validation<jupyter_code>url17 = 'https://www.cdc.gov/brfss/annual_data/2017/llcp_varlayout_17_onecolumn.html'
url16 = 'https://www.cdc.gov/brfss/annual_data/2016/LLCP_VarLayout_16_OneColumn.html'

filename16 = 'LLCP2016ASC.zip'
filename17 = 'LLCP2017ASC.zip'

tables = pd.read_html(url17)
layout = tables[1]
layout.index = layout['Variable Name']
layout
names = [ 'SEX', 'HTM4', 'WTKG3', 'INCOME2', '_LLCPWT' , '_AGEG5YR', '_VEGESU1']
colspecs = []
for name in names:
    start, _, length = layout.loc[name]
    colspecs.append((start-1, start+length-1))
    
colspecs
brfss = pd.read_fwf(filename17,
                    colspecs=colspecs, 
                    names=names,
                    compression='zip',
                    nrows=None)

brfss.head()
brfss.shape
brfss['SEX'].value_counts().sort_index()
brfss['SEX'].replace([9], np.nan, inplace=True)
brfss['INCOME2'].value_counts().sort_index()
brfss['INCOME2'].replace([77, 99], np.nan, inplace=True)
brfss['WTKG3'] /= 100
brfss['WTKG3'].describe()
weight = brfss['WTKG3']
weight.nsmallest(10)
weight.nlargest(10)
height = brfss['HTM4']
height.nsmallest(10)
height.nlargest(10)
brfss['HTM4'].describe()
brfss['_LLCPWT'].describe()
brfss['_AGEG5YR'].describe()
brfss['_AGEG5YR'].value_counts().sort_index()
brfss['_AGEG5YR'].replace([14], np.nan, inplace=True)
brfss['_VEGESU1'] /= 100
Pmf(brfss['_VEGESU1']).plot()
bogus = brfss['_VEGESU1'] > 15
brfss.loc[bogus, '_VEGESU1'] = np.nan
brfss['_VEGESU1'].describe()<jupyter_output><empty_output><jupyter_text>Add a height group column<jupyter_code>bins = np.arange(0, height.max(), 10)
brfss['_HTMG10'] = pd.cut(brfss['HTM4'], bins=bins, labels=bins[:-1]).astype(float)
brfss._HTMG10.dtype
lower = np.arange(15, 85, 5)
upper = lower + 4
lower[1]= 18
lower = pd.Series(lower, index=range(len(lower)))
lower
upper[-1] = 99
upper = pd.Series(upper, index=range(len(upper)))
upper
midpoint = (lower + upper) / 2
midpoint
age_code = brfss['_AGEG5YR']
brfss['AGE'] = midpoint[age_code].values
brfss['AGE'].describe()
def randint(lower, upper):
    for low, high in zip(lower, upper+1):
        try:
            yield np.random.randint(low, high)
        except ValueError:
            yield np.nan<jupyter_output><empty_output><jupyter_text>Resample<jupyter_code>np.random.seed(17)
sample = utils.resample_rows_weighted(brfss, '_LLCPWT')[:100000]
sample.head(10)
!rm brfss.hdf5
sample.to_hdf('brfss.hdf5', 'brfss')
%time brfss = pd.read_hdf('brfss.hdf5', 'brfss')
brfss.shape
brfss.head()
brfss.describe()
%time gss = pd.read_hdf('gss.hdf5', 'gss')
gss.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16423357664233576, 'fraction_numerical': 0.038487060384870604, 'mean_word_length': 5.387711864406779, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 19, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '75208', 'license': 'MIT', 'n_tokens_mistral': 1309, 'n_tokens_neox': 1230, 'n_words': 195, 'path': 'brfss_validate.ipynb', 'repo_name': 'AllenDowney/ExploratoryDataAnalysis', 'size': '3020'}