<jupyter_start><jupyter_text>In this file, we will extract the data present in all the pages page 1, 2, 3, 4Also, we extract the data present in the lab name which is a link => so extract from that link too<jupyter_code>import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
lis = [dict()]
bad_chars = [';',',', '.', '(', ')','-']
for t in range(4):
    url = "https://www.ams.usda.gov/rules-regulations/hemp/dea-laboratories?field_lab_location_administrative_area=All&page="+str(t)
    req = requests.get(url)
    soup = BeautifulSoup(req.text,"html.parser")
    #print(soup.prettify())
    tb = soup.find("tbody")

    for t in tb.find_all("tr"):
      i=0;
      dic = dict()
      for d in t.find_all("td"):
        if i==0:
          dic["Lab Name"] = d.text
          for l in d.find_all("a",href=True):
            if(l.text):
              link = l['href']
              link = "https://www.ams.usda.gov/"+link
          
      
          req2 = requests.get(link)
          soup2 = BeautifulSoup(req2.text,"html.parser")
          #print("prettify")
      
          lc =soup2.find("div",attrs={"class":"field field--name-field-lab-location field--type-address field--label-above"})
          address = lc.find("span",attrs={"class":"address-line1"})
          #print(address.text)

          locality =lc.find("span",attrs={"class":"locality"})
          #print(locality.text)

          admin =lc.find("span",attrs={"class":"administrative-area"})
          #print(admin.text)

          postal =lc.find("span",attrs={"class":"postal-code"})
          #print(postal.text)

          country =lc.find("span",attrs={"class":"country"})
          #print(country.text)

          dic["Lab Location"] = address.text + " " + locality.text + " " + admin.text + " " + postal.text + " " + country.text 
      
          if(soup2.find("div",attrs={"class":"field field--name-field-phone-number field--type-string field--label-above"})):
            dic["Phone Number"] = soup2.find("div",attrs={"class":"field field--name-field-phone-number field--type-string field--label-above"}).find("div",attrs={"class":"field__item"}).text
          else:
            dic["Phone Number"]= "No Content"
          
          if(soup2.find("div",attrs={"class":"field field--name-field-contact field--type-string field--label-above"})):
            dic["Contact"] = soup2.find("div",attrs={"class":"field field--name-field-contact field--type-string field--label-above"}).find("div",attrs={"class":"field__item"}).text
          else:
            dic["Contact"] = "No content"

          if(soup2.find("div",attrs={"class":"field field--name-field-contact-email field--type-email field--label-above"})):
            dic["Email"] = soup2.find("div",attrs={"class":"field field--name-field-contact-email field--type-email field--label-above"}).find("div",attrs={"class":"field__item"}).text
          else:
            dic["Email"] = "No Content"

    
          #print(link)
          i =(i+1)%3
        elif i==1:
          #print(d.text)
          dic["City"] = d.text

      lis.append(dic)

del lis[0:1]
print(len(lis))
df = pd.DataFrame(lis)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18612680477087257, 'fraction_numerical': 0.007532956685499058, 'mean_word_length': 2.684393063583815, 'pattern_counts': {'":': 15, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '345858', 'license': 'Unlicense', 'n_tokens_mistral': 1058, 'n_tokens_neox': 1026, 'n_words': 177, 'path': 'Beautiful Soup-3.ipynb', 'repo_name': 'Akhilesh53/WebScrapping', 'size': '3186'}