<jupyter_start><jupyter_text>Web Scraping<jupyter_code>import requests
from bs4 import BeautifulSoup
page = requests.get("https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/Fallzahlen.html")
soup = BeautifulSoup(page.content, 'html.parser')
html_table = soup.find('table')
all_rows = html_table.find_all('tr')
final_data_list = []
for pos, rows in enumerate(all_rows):
    #print(rows)
    col_list = [each_col.get_text(strip=True) for each_col in rows.find_all('td')]
    final_data_list.append(col_list)
pd.DataFrame(final_data_list).dropna().rename(columns={0:'State'})<jupyter_output><empty_output><jupyter_text>REST API CALLS<jupyter_code>data = requests.get("https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_Landkreisdaten/FeatureServer/0/query?where=1%3D1&outFields=*&outSR=4326&f=json")
import json
json_object = json.loads(data.content)
json_object.keys()
full_list = []
for pos, each_dict in enumerate(json_object['features'][:]):
    full_list.append(each_dict['attributes'])
pd.DataFrame(full_list)<jupyter_output><empty_output><jupyter_text>API access via REST Service<jupyter_code>import requests
url_endpoint = 'https://api.smartable.ai/coronavirus/stats/IN'
headers = {
# Request headers
'Cache-Control': 'no-cache',
'Subscription-Key': '<KEY>',
}
response = requests.get(url_endpoint, headers = headers)
print(response)
US_dict = json.loads(response.content)
with open('../data/raw/SMARTABLE/IN_data.txt', 'w') as outfile:
    json.dump(US_dict, outfile, indent=2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14970836033700582, 'fraction_numerical': 0.008425145819831496, 'mean_word_length': 7.7727272727272725, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '78391', 'license': 'FTL', 'n_tokens_mistral': 584, 'n_tokens_neox': 576, 'n_words': 76, 'path': 'notebooks/Data_Understanding.ipynb', 'repo_name': 'kinger-ml/eds-covid19', 'size': '1570'}