<jupyter_start><jupyter_text>Acquire data & data structures<jupyter_code>import requests
import pandas as pd<jupyter_output><empty_output><jupyter_text>Internet  Check the response status code - status code 200: the request response cycle was successful - any other status code: it didn't work (e.g., 404 = page not found) - Convert content to utf-8 if necessary<jupyter_code>def connect(url, decode='utf-8'):
    response = requests.get(url)
    if response.status_code == 200:
        print('successfully connected, response code: {}'.format(response.status_code))
    else:
        print('connection failed')
    return response.content.decode(decode)
url = 'http://www.lauthom.nl/search/tools'
content = connect(url)
content[:500]<jupyter_output><empty_output><jupyter_text>JSON<jupyter_code>import json<jupyter_output><empty_output><jupyter_text>json.loads recursively decodes a string in JSON format into equivalent python objects - data_string's outermost element is converted into a python list - the first element of that list is converted into a dictionary - the key of that dictionary is converted into a string - the value of that dictionary is converted into a list of two integer elements<jupyter_code>data_string = '[{"b": [2, 4], "c": 3.0, "a": "A"}]'

python_data = json.loads(data_string)
print('{}\n{}\n{}\n{}\n{}\n{}'.format(type(data_string), type(python_data), python_data, python_data[0], python_data[0]['b'], python_data[0]['b'][1]))<jupyter_output><empty_output><jupyter_text>json.dumps and json.loads<jupyter_code>JSON_string = "JSON throws exception when not in correct format"
print(JSON_string)

# Stringify strings
JSON_stringified = json.dumps(JSON_string)
print(JSON_stringified)

# Correct
json.loads(JSON_stringified)

# JSONDecodeError 
# json.loads(JSON_string)<jupyter_output><empty_output><jupyter_text>requests & JSON<jupyter_code>address = 'Amsterdam, Netherlands'
url = 'https://maps.googleapis.com/maps/api/geocode/json?address={}'.format(address)
response = requests.get(url).json()
type(response), response<jupyter_output><empty_output><jupyter_text>Get JSON formatted content<jupyter_code>def get_json(url, decode='utf-8'):
    try:
        response = requests.get(url)
        if not response.status_code == 200:
            print('HTTP error, response code: {}'.format(response.status_code))
        else:
            try:
                response_data = response.json()
            except:
                print("response not in valid JSON format")
    except:
        print('something went wrong with requests.get')
        
    return response_data
response_data = get_json(url)
response_data<jupyter_output><empty_output><jupyter_text>Get address, latitude, longitude<jupyter_code>def get_lat_lng(url):
    response = get_json(url)
    result = response['results'][0]
    formatted_address = result['formatted_address']
    lat = result['geometry']['location']['lat']
    lng = result['geometry']['location']['lng']
    return formatted_address, lat, lng
get_lat_lng(url)
address = 'London Business School'
url = 'https://maps.googleapis.com/maps/api/geocode/json?address={}'.format(address)
get_lat_lng(url)<jupyter_output><empty_output><jupyter_text>Get list of addresses with lat, lon<jupyter_code>def get_lat_lng_list(url):
    response = get_json(url)
    result_list = []
    for result in response['results']:
        formatted_address = result['formatted_address']
        lat = result['geometry']['location']['lat']
        lng = result['geometry']['location']['lng']
        result_list.append((formatted_address, lat, lng))
    return result_list
address = 'Baker Street'
url = 'https://maps.googleapis.com/maps/api/geocode/json?address={}'.format(address)
get_lat_lng_list(url)<jupyter_output><empty_output><jupyter_text>XML - library lxml - deals with converting an XML-string to python objects and vice versa<jupyter_code>from lxml import etree
data_string = """
<Bookstore>
   <Book ISBN="ISBN-13:978-1599620787" Price="15.23" Weight="1.5">
      <Title>New York Deco</Title>
      <Authors>
         <Author Residence="New York City">
            <First_Name>Richard</First_Name>
            <Last_Name>Berenholtz</Last_Name>
         </Author>
      </Authors>
   </Book>
   <Book ISBN="ISBN-13:978-1579128562" Price="15.80">
      <Remark>
      Five Hundred Buildings of New York and over one million other books are available for Amazon Kindle.
      </Remark>
      <Title>Five Hundred Buildings of New York</Title>
      <Authors>
         <Author Residence="Beijing">
            <First_Name>Bill</First_Name>
            <Last_Name>Harris</Last_Name>
         </Author>
         <Author Residence="New York City">
            <First_Name>Jorg</First_Name>
            <Last_Name>Brockmann</Last_Name>
         </Author>
      </Authors>
   </Book>
</Bookstore>
"""
root = etree.XML(data_string)
root.tag, type(root.tag)
print(etree.tostring(root, pretty_print=True).decode("utf-8"))<jupyter_output><empty_output><jupyter_text>Iterating over complete XML tree<jupyter_code>for element in root.iter():
    print(element)<jupyter_output><empty_output><jupyter_text>Iterate over children in subtree, accessing tags<jupyter_code>for child in root:
    print(child, child.tag)<jupyter_output><empty_output><jupyter_text>Iterate to get specific tags and data1. author tags are accessed2. For each author tag, the .find function accesses the First_Name and Last_Name tags3. The .find function only looks at the children, not other descendants, so be careful!4. The .text attribute prints the text in a leaf node<jupyter_code>for element in root.iter('Author'):
    print(element.find('First_Name').text, element.find('Last_Name').text)<jupyter_output><empty_output><jupyter_text>Filter values of attributese.g. find the first name of the author of a book that weighs 1.5 oz<jupyter_code>root.find('Book[@Weight="1.5"]/Authors/Author/First_Name').text<jupyter_output><empty_output><jupyter_text>Exchange rates from  XE.com<jupyter_code>url = 'https://www.xe.com/currencyconverter/convert/?Amount=1&From=USD&To=EUR'<jupyter_output><empty_output><jupyter_text>BeautifulSoup<jupyter_code>from bs4 import BeautifulSoup
def result_page(url, keywords=''):
    response = requests.get(url + keywords)
    if not response.status_code == 200:
        return None
    return BeautifulSoup(response.content, 'lxml')
def get_data(url, keywords='', selector=''):
    rate_list = []
    try:
        results_page = result_page(url, keywords)
        rates = results_page.find_all('td', class_='rateCell')
        
        for rate in rates:
            rate_ = rate.get_text()
            try:
                currency = rate.find('a').get('rel')[0][:7]
                rate_list.append((currency, rate_))
            except:
                currency = ''
            
        return rate_list
    except:
        return None
pd.DataFrame(get_data(url))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12372025955299207, 'fraction_numerical': 0.011968276856524874, 'mean_word_length': 4.298701298701299, 'pattern_counts': {'":': 3, '<': 99, '<?xml version=': 0, '>': 99, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '35915', 'license': 'MIT', 'n_tokens_mistral': 2263, 'n_tokens_neox': 2177, 'n_words': 485, 'path': '_acquire_data/acquire_data_.ipynb', 'repo_name': 'aixpact/data-science', 'size': '6935'}