<jupyter_start><jupyter_text>Strategy for scraping- All of the required data is contained in a script tag which begins with the text "window.__INITIAL_STATE__"- First I have extracted all script tags from the soup- Then I have extracted the required script tag (by checking if it begins with the "window.__INITIAL_STATE__" text)- Then I perform a bit of cleaning before loading the required data as json Note - Each of the script tags only contain 20 of the items even though we see 30 items per page. But the remaining items are available by making requests to the next page<jupyter_code>def extract_json(soup):
    all_scripts=soup.find_all("script")
    
    #extracting the script tag as a string (storing in a list)
    all_med_data=[]
    for script in all_scripts:
        # to check if a script tag can be converted to a string and if the string has the "window.__INITIAL__STATE__"
        if (script.string) and ("window.__INITIAL_STATE__" in script.string):
            all_med_data.append(script.string)
    json_data=all_med_data[0].replace("\n    window.__INITIAL_STATE__ = ",""
                       ).replace(";\n    window.__STATUS_CODE__ = null;\n","")
    json_data=json.loads(json_data)
    
    return(json_data['allMedicinePageReducer']['data'])
extract_json(soup)<jupyter_output><empty_output><jupyter_text>Stuff to record for each card - 1. Id and Skuid (for future fetching of data)2. Name3. Prescription req    4. is_discontinued5. manufacturer6. Type7. Pack size8. short_composition9. rx_required.header<jupyter_code>%%time
for i in URLS:
    try:
        page_req=requests.get(URL)
        soup=BeautifulSoup(page_req.content, "html.parser")
        print("present")
    except:
        print(f{"URL not present",{i}})
pd.DataFrame({str(i['id']):(str(i['sku_id']),i['name'],i['manufacturer_name'],
  i['type'],i['pack_size_label'],str(i['price']),
  i['rx_required']['header'],
  i['short_composition'],str(i['is_discontinued'])) for i in extract_json(soup)['skus']}).T<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11248149975333005, 'fraction_numerical': 0.006906758756783424, 'mean_word_length': 4.322834645669292, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '178933', 'license': 'MIT', 'n_tokens_mistral': 638, 'n_tokens_neox': 622, 'n_words': 192, 'path': 'Tata1mg Scraper.ipynb', 'repo_name': 'hippocampus13/IndianMedicineDB', 'size': '2027'}