<jupyter_start><jupyter_text>Set Params<jupyter_code>with open(r'data/urls.txt') as f:
    urls = f.read().splitlines()
    
urls[0]<jupyter_output><empty_output><jupyter_text>Get Business Data<jupyter_code># Warning! Monitor the loop for Captchas

data_json = [] # Collect results

for url in tqdm(urls):
    
    time.sleep(10+10*random.random())
    
    ua = UserAgent() # Generate random user to make it seem like the task is distributed across multiple computers
    user_agent = {'User-agent': ua.random}
    response  = requests.get(url, headers = user_agent)
    soup = bs4.BeautifulSoup(response.text)    
    
    # Print information for tracking
#     page_text = soup.find('div',{"class":"page-of-pages arrange_unit arrange_unit--fill"}).get_text().strip() # Get 'Page <current> of <total>'
#     num_pages = int(re.search('(?<=of) [0-9]*', page_text).group(0).strip()) # Get total number of pages
#     num_reviews = soup.find('span',{"class":"review-count rating-qualifier"}).get_text().strip() # Get total number of reviews
#     business_id = soup.find("div", {"class": "lightbox-map hidden"}).get("data-business-id")
#     print(*[url, business_id, 'Index: '+str(url_index), num_reviews], sep="\n") # Quality Control
        
    data_dict = get_business_data(soup)
    data_json.append(data_dict) # Collect data
if save:
    filename = "business"
    with open(f'data/{filename}.json', 'w') as f:
        for review in tqdm(data_json):
            json.dump(review, f)
            f.write('\n')
    print('Complete')<jupyter_output>Complete<jupyter_text>Check<jupyter_code>business_full_bag = db.read_text("data/business.json").map(json.loads) # Loads the json file as a dask bag
business_tuple = business_full_bag.take(10000) # Takes the first 10000 entries of the dask bag and stores as a tuple
len(business_tuple)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14316353887399463, 'fraction_numerical': 0.010187667560321715, 'mean_word_length': 4.286118980169972, 'pattern_counts': {'":': 3, '<': 15, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '611691', 'license': 'MIT', 'n_tokens_mistral': 640, 'n_tokens_neox': 623, 'n_words': 141, 'path': 'get-business-data-from-yelp.ipynb', 'repo_name': 'harrisonized/analyzing-yelp-reviews-for-climbing-gyms-nlp', 'size': '1865'}