<jupyter_start><jupyter_text>Airbnb Data<jupyter_code>import numpy as np
import os
import pandas as pd
import matplotlib.pyplot as plt
from urllib3.exceptions import ProtocolError
import pprint

from pandas.io.json import json_normalize
import pandas as pd
from urllib3.exceptions import ProtocolError
api_base = 'https://www.airbnb.com/api/v2/explore_tabs?version=1.3.8&_format=for_explore_search_web&experiences_per_grid=20&items_per_grid=18&guidebooks_per_grid=20&auto_ib=true&fetch_filters=true&has_zero_guest_treatment=false&is_guided_search=true&is_new_cards_experiment=true&luxury_pre_launch=false&query_understanding_enabled=true&show_groupings=true&supports_for_you_v3=true&timezone_offset=-360&client_session_id=82674acc-b274-41dc-8b00-499d5c2fea44&metadata_only=false&is_standard_search=true&refinement_paths%5B%5D=%2Fhomes&selected_tab_id=home_tab&place_id=ChIJ5S-raZElv0cR8HcqSvxgJwQ&allow_override%5B%5D=&s_tag=kke9hwU1&screen_size=medium&query=Cologne%2C+Germany&_intents=p1'
api_key = '&key=d306zoyjsyarp7ifhu67rjxn52tv0t20&currency=USD&locale=en'
api_section_offset = '&section_offset=4'
import requests
import json
retry_lvl = [100, 0.1]<jupyter_output><empty_output><jupyter_text>Key<jupyter_code>from bs4 import BeautifulSoup
airbnb_home = requests.get("http://airbnb.com")

soup = BeautifulSoup(airbnb_home.content, "html.parser")

metatags = soup.find_all('meta', id="_bootstrap-layout-init")
metacontents = [metatag["content"] for metatag in metatags]

metajson = json.loads(metacontents[0])

metajson

api_key = metajson['api_config']['key']
api_key = '&key='+api_key
print(api_key)<jupyter_output>&key=d306zoyjsyarp7ifhu67rjxn52tv0t20<jupyter_text>Explore API<jupyter_code>rsp = requests.get(api_base+api_key)
rsp_json = rsp.json()
len(rsp_json)
rsp_json.keys()
"pagination_metadata": {
        "has_next_page": true,
        "items_offset": 18,
        "section_offset": 4,
        "search_session_id": "eee6b55b-100d-4314-800a-393bec147f25"
      }
rsp_json['explore_tabs'][0]['pagination_metadata']['has_next_page']
rsp_json['explore_tabs'][0]['home_tab_metadata'].keys()
len(rsp_json['explore_tabs'][0]['sections'])
rsp_json['explore_tabs'][0]['sections'][3].keys()
len(rsp_json['explore_tabs'][0]['sections'][3]['listings'])
rsp_json['explore_tabs'][0]['sections'][3]['listings'][0];
flatten_json(rsp_json['explore_tabs'][0]['sections'][3]['listings'][0]);<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code># fields

listing_fields = [
'bathrooms',
'bedrooms',
'beds',
'person_capacity',
# 'host_languages',
'id',
'is_business_travel_ready',
'is_fully_refundable',
'is_host_highly_rated',
'is_rebookable',
'is_superhost',
'lat',
'lng',
'picture_count',
'preview_amenities',
'reviews_count',
'star_rating',
'tier_id'
]

pricing_quote_rate_with_service_fee_fields = [
'amount',
'currency'
]

fields_default = ['listing_'+ field for field in listing_fields] + ['pricing_quote_rate_with_service_fee_'+field for field in pricing_quote_rate_with_service_fee_fields] + ['pricing_quote_rate_type']
fields_default
def listing_parser(listing, fields = None):
    if fields == None:
        fields = ['listing_bathrooms', 'listing_bedrooms', 'listing_beds', 'listing_person_capacity', 'listing_host_languages', 'listing_id', 'listing_is_business_travel_ready', 'listing_is_fully_refundable', 'listing_is_host_highly_rated', 'listing_is_rebookable', 'listing_is_superhost', 'listing_lat', 'listing_lng', 'listing_picture_count', 'listing_preview_amenities', 'listing_reviews_count', 'listing_star_rating', 'listing_tier_id', 'pricing_quote_rate_with_service_fee_amount', 'pricing_quote_rate_with_service_fee_currency', 'pricing_quote_rate_type']
    
    return pd.DataFrame(listing)[fields_default]

        
    
    
def next_indicator(response_json):
    return response_json['explore_tabs'][0]['pagination_metadata']['has_next_page']
def flatten_json(y):
    out = {}

    def flatten(x, name=''):
        if type(x) is dict:
            for a in x:
                flatten(x[a], name + a + '_')
        elif type(x) is list:
            i = 0
            for a in x:
                flatten(a, name + str(i) + '_')
                i += 1
        else:
            out[name[:-1]] = x

    flatten(y)
    return out<jupyter_output><empty_output><jupyter_text>Demo Scraping<jupyter_code>next_sec = True
sec_off = 0
rsp = requests.get(api_base+api_key+'&section_offset=' + str(sec_off))
rsp_json = rsp.json()

listing_array = [flatten_json(item) for item in rsp_json['explore_tabs'][0]['sections'][3]['listings'] ]
df_total = listing_parser(listing_array)

df_total = pd.concat([df, df_total], join='inner')

next_sec = next_indicator(rsp_json)

sec_off = sec_off + 1
while next_sec == True:
    
    try_flag = 0

    while True and (try_flag < retry_lvl[0]):
        try:
            with requests.Session() as s:
                rsp = s.get(api_base+api_key+'&section_offset='+str(sec_off))
        except requests.exceptions.RequestException as err:
            print('request error', err)
            try_flag = try_flag + 1
            sleep(try_flag * retry_lvl[1])
            pass
        else:
            break

    rsp_json = rsp.json()
    

    listing_array = [flatten_json(item) for item in rsp_json['explore_tabs'][0]['sections'][0]['listings'] ]

    df = listing_parser(listing_array)
    df_total = pd.concat([df, df_total], join='inner')
    
    sec_off = sec_off + 1
    
    next_sec = next_indicator(rsp_json)
    
df_total
df_total.to_csv('data/cologne.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13333333333333333, 'fraction_numerical': 0.023423423423423424, 'mean_word_length': 5.861557478368356, 'pattern_counts': {'":': 5, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '41705', 'license': 'MIT', 'n_tokens_mistral': 2154, 'n_tokens_neox': 2084, 'n_words': 282, 'path': 'airbnb/pretest.ipynb', 'repo_name': 'emptymalei/data-acquisition', 'size': '5550'}