<jupyter_start><jupyter_text><jupyter_code>response = requests.get("https://webscraper.io/test-sites/\
e-commerce/allinone")
doc = pq(response.text)<jupyter_output><empty_output><jupyter_text><jupyter_code>dataList = []
doc.make_links_absolute(base_url=response.url)
for eachMainDoc in doc("#side-menu > li:nth-child(n+2) > a").items():
  # print(eachMainDoc.attr("href"))
  mainRes = requests.get(eachMainDoc.attr("href"))
  mainDoc = pq(mainRes.text)
  mainDoc.make_links_absolute(base_url=mainRes.url)
  for eachSubDoc in mainDoc("#side-menu > li.active > ul > li > a").items():
    print(eachSubDoc.attr("href"))
    subRes = requests.get(eachSubDoc.attr("href"))
    subDoc = pq(subRes.text)
    #level 2
    for eachItem in subDoc(".test-site .thumbnail").items():
      dataDict = {}
      dataDict["title"] = eachItem("h4>.title").text()
      dataDict["price"] = eachItem("h4.price").text()
      dataDict["stars"] = len(eachItem("span.glyphicon-star"))
      dataList.append(dataDict)
      # print(eachItem("h4>.title").text())
      # print(eachItem("h4.price").text())
      # print(len(eachItem("span.glyphicon-star")))
      # print((eachItem(".ratings>p:nth-child(2)").attr("data-rating"))
dataList<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18971061093247588, 'fraction_numerical': 0.005627009646302251, 'mean_word_length': 4.956937799043062, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '640798', 'license': 'MIT', 'n_tokens_mistral': 457, 'n_tokens_neox': 450, 'n_words': 51, 'path': 'E-commerce.ipynb', 'repo_name': 'kaiicheng/Python-Crawler', 'size': '1244'}