<jupyter_start><jupyter_text>Authentication<jupyter_code>base_url = 'https://api.hackathon-03.batch2.lisbondatascience.org/'
auth_req = base_url+'api/token-auth'
response = requests.get(auth_req,params={'username':'ldsa_student'})
token = response.json()
token<jupyter_output><empty_output><jupyter_text>Get ALL THE DATA!<jupyter_code>headers={'Authorization':'token '+ token['token']}
status=200
page=1
while status==200:
    query = base_url+'api/data?page='+str(page)+'&page_size=2000'
    query_resp = requests.get(query, headers=headers)
    query_result = query_resp.json()
    if query_resp.status_code ==200:
        page_df=pd.DataFrame(query_result['results'])
        if page==1:
            total=pd.DataFrame(page_df)
        else:
            total=total.append(page_df)
    status=query_resp.status_code
    page+=1<jupyter_output><empty_output><jupyter_text>Export<jupyter_code>total.to_csv('data/api/api_data.csv')<jupyter_output><empty_output><jupyter_text>wrangling<jupyter_code>total = pd.read_csv('data/api/api_data.csv')
total['emergencystate_mode'] = total['emergencystate_mode'].fillna(0)
total.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 100000 entries, 0 to 99999
Data columns (total 12 columns):
Unnamed: 0                   100000 non-null int64
amt_req_credit_bureau_qrt    86579 non-null float64
apartments_avg               49443 non-null float64
days_birth                   100000 non-null int64
emergencystate_mode          100000 non-null object
flag_document_12             100000 non-null bool
flag_document_13             100000 non-null bool
flag_document_9              100000 non-null bool
name_education_type          100000 non-null object
name_income_type             100000 non-null object
obs_30_cnt_social_circle     100000 non-null int64
sk_id_curr                   100000 non-null int64
dtypes: bool(3), float64(2), int64(4), object(3)
memory usage: 7.2+ MB
{'fraction_non_alphanumeric': 0.10904392764857881, 'fraction_numerical': 0.06925064599483204, 'mean_word_length': 4.108179419525066, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '490916', 'license': 'MIT', 'n_tokens_mistral': 751, 'n_tokens_neox': 643, 'n_words': 109, 'path': 'Api_data.ipynb', 'repo_name': 'orlandocosta76/subprime3ever', 'size': '1935'}