<jupyter_start><jupyter_text>最简单的爬虫> 1. 建立请求* 解析数据* 存储成json<jupyter_code>import requests
from bs4 import BeautifulSoup<jupyter_output><empty_output><jupyter_text>请求<jupyter_code># hide
res = requests.get('http://www.baidu.com')
res,res.text<jupyter_output><empty_output><jupyter_text>解析<jupyter_code># hide
soup = BeautifulSoup(res.text)
soup
# hide
a_htmls = soup.find_all('a',class_='mnav')
a_htmls
# hide
for a in a_htmls:
    print(a['href'])<jupyter_output>http://news.baidu.com
http://www.hao123.com
http://map.baidu.com
http://v.baidu.com
http://tieba.baidu.com<jupyter_text>保存<jupyter_code># hide
with open('./data/00_links.txt', 'w') as f:
    for a in a_htmls:
        f.write(a['href']+'\n')<jupyter_output><empty_output><jupyter_text>读取<jupyter_code># hide
with open('./data/00_links.txt', 'r') as f:
    print(f.read())<jupyter_output>http://news.baidu.com
http://www.hao123.com
http://map.baidu.com
http://v.baidu.com
http://tieba.baidu.com
{'fraction_non_alphanumeric': 0.18134171907756813, 'fraction_numerical': 0.012578616352201259, 'mean_word_length': 5.870503597122302, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '608564', 'license': 'Apache-2.0', 'n_tokens_mistral': 456, 'n_tokens_neox': 446, 'n_words': 54, 'path': '00_Getting_Started.ipynb', 'repo_name': 'neo4dev/crawler_from_scratch', 'size': '954'}