<jupyter_start><jupyter_text>Crawling Data from Public API<jupyter_code>import urllib.request as ul
import json
import sys
import io
import pandas as pd<jupyter_output><empty_output><jupyter_text>Crawling a Json format dataset and convert it to DICT<jupyter_code>url = "https://data.cms.gov/data-api/v1/dataset/5fccd951-9538-48a7-9075-6f02b9867868/data"
request = ul.Request(url)
response = ul.urlopen(request)
rescode = response.getcode()

# if the request is accepted, rescode = 200
if(rescode == 200):
    responseData = response.read()
    rDD = json.loads(responseData)<jupyter_output><empty_output><jupyter_text>* url is the API url   Crawling unlimited number of rows  * Resulting format is pandas Data Frame* url: API url* limit_num: the number of rows you want to crawl at once* maxLimit: the last number of rows you want to crawl (how many rows in total)<jupyter_code>def crawl(url, limit_num=None, maxLimit=None):
    
    limitParam = "?limit="
    offsetParam = "&offset="
    limit = str(limit_num)
    offset = 0
    totalURL = url+limitParam+limit+offsetParam+str(offset)
    rDD = []

    while limit_num <= maxLimit:
        request = ul.Request(totalURL)
        response = ul.urlopen(request)
        rescode = response.getcode()
        if(rescode == 200):
            responseData = response.read()
            rDD.append(json.loads(responseData))
        
        limit_num += 1000
        offset += 1000
        
        # show process
        count = round((limit_num/maxLimit)*100, 1)
        if limit_num in list(range(0, maxLimit, 10000)):
            print("process: ", count, "%")
            
    return rDD
def combData(url, limit_num, maxLimit):
    
    result = crawl(url, limit_num, maxLimit)
    
    result_df = pd.DataFrame()
    for i in range(len(result)):
        rDD_df = pd.DataFrame(result[i])
        result_df = pd.concat([result_df, rDD_df])
    
    return result_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10591259640102828, 'fraction_numerical': 0.027763496143958868, 'mean_word_length': 3.2396514161220042, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '249819', 'license': 'MIT', 'n_tokens_mistral': 679, 'n_tokens_neox': 632, 'n_words': 160, 'path': 'Data_Crawling_API.ipynb', 'repo_name': 'hshehjue/My_Functions', 'size': '1945'}