<jupyter_start><jupyter_text>基本爬取例子 爬取百度搜索结果<jupyter_code>import requests
keyword = 'Python'

try:
    kv = {'wd':keyword}
    r = requests.get('http://www.baidu.com/s', params = kv)
    r.raise_for_status()
    r.encoding = r.apparent_encoding
    print(len(r.text))
except:
    print("Error")<jupyter_output><empty_output><jupyter_text>更改头部信息,获取京东商品<jupyter_code>import requests
url = 'https://www.aptagen.com/apta-index/'
try:
    kv={'user-agent':'Mozilla/5.0'}
    r=requests.get(url,headers=kv)
    r.raise_for_status()
    r.encoding=r.apparent_encoding
    print('爬取成功')
except:
    print("爬取失败")<jupyter_output>爬取成功<jupyter_text>保存图片<jupyter_code>import requests, os
url = 'https://www.aptagen.com/wp-content/uploads/358.png'
root = '/Users/jiamingxu/Downloads/'
path = root + url.split('/')[-1]
try:
    if not os.path.exists(root):
        os.mkdir(root)
    if not os.path.exists(path):
        r = requests.get(url, headers={'user-agent':'Chrome/16.0'})
        with open(path, 'wb') as f:
            f.write(r.content)
            f.close()
            print('File saved')
    else:
        print('File exists')
        
except:
    print('Failed')
r.content<jupyter_output><empty_output><jupyter_text>查询ip归属地<jupyter_code>import requests
url = 'https://m.ip138.com/iplookup.asp?ip='
kv={'user-agent':'Mozilla/5.0'}
r = requests.get(url + '192.168.127.12', headers=kv)
r.status_code
r.encoding = r.apparent_encoding
r.text[100:1000]<jupyter_output><empty_output><jupyter_text>解析HTML页面 - Beautiful Soup<jupyter_code>import requests
import requests
url = 'https://www.aptagen.com/apta-index/'
try:
    kv={'user-agent':'Mozilla/5.0'}
    r=requests.get(url,headers=kv)
    r.raise_for_status()
    r.encoding=r.apparent_encoding
    demo = r.text
    print('爬取成功')
except:
    print("爬取失败")
from bs4 import BeautifulSoup
import re
soup = BeautifulSoup(demo, 'html.parser')
soup.a.parent.parent.name
tag = soup.a
tag.attrs['href']
type(tag)
soup.a.string
type(soup.p.string)
type(soup.p.string)
for i in soup.p.contents:
    print(i)
soup.head.contents
soup.body.contents
len(soup.body.contents)
soup.body.contents[1]
soup.a.next_sibling.next_sibling
soup.a.previous_sibling.previous_sibling
for tag in soup(True):
    print(tag.name)
for tag in soup(re.compile('p')):
    print(tag.name)
for tag in soup('p', 'course'):
    print(tag)
for tag in soup(id='link1'):
    print(tag)<jupyter_output><a class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">Basic Python</a><jupyter_text>中国大学排名爬取<jupyter_code>import requests
from bs4 import BeautifulSoup
import bs4

def getHTMLText(url):
    try:
        kv={'user-agent':'Mozilla/5.0'}
        r=requests.get(url, headers=kv, timeout = 30)
        r.raise_for_status()
        r.encoding=r.apparent_encoding
        print('爬取成功')
        return r.text
    
    except:
        return ""
        
def fillUnivList(ulist, html):
    soup = BeautifulSoup(demo, 'html.parser')
    for tr in soup.find('tbody').children:
        if isinstance(tr, bs4.element.Tag):
            tds = tr('td')
            ulist.append([tds[0].string.split()[0],str(tds[1].find('a').string),float(tds[4].string)])            


def printUnivList(ulist, num):
    tplt = "{0:^10}\t{1:{3}^10}\t{2:^10}"   # 1:{3}表示使用format里第三个变量
    print(tplt.format('排名','学校名称','总分', chr(12288))) # chr(12288) = 中文空格
    for i in range(num):
        u = ulist[i]
        print(tplt.format(u[0],u[1],u[2], chr(12288))) 
    #print("Suc" + str(num))
    
    
    
def main():
    global uinfo
    uinfo = []
    url = 'https://www.shanghairanking.cn/rankings/bcur/2021'
    html = getHTMLText(url)
    fillUnivList(uinfo, html)
    printUnivList(uinfo, 20)  # first 20 universities
    
main()<jupyter_output>爬取成功
    排名    	　　　学校名称　　　	    总分    
    1     	　　清华大学 　　　	  969.2   
    2     	　　北京大学 　　　	  855.3   
    3     	　　浙江大学 　　　	  768.7   
    4     	　上海交通大学 　　	  723.4   
    5     	　　南京大学 　　　	  654.8   
    6     	　　复旦大学 　　　	  649.7   
    7     	中国科学技术大学 　	  577.0   
    8     	　华中科技大学 　　	  574.3   
    9     	　　武汉大学 　　　	  567.9   
    10    	　西安交通大学 　　	  537.9   
    11    	　哈尔滨工业大学 　	  522.6   
    12    	　　中山大学 　　　	  519.3   
    13    	　北京师范大学 　　	  518.3   
    14    	　　四川大学 　　　	  516.6   
    15    	北京航空航天大学 　	  513.8   
    16    	　　同济大学 　　　	  508.3   
    17    	　　东南大学 　　　	  488.1   
    18    	　中国人民大学 　　	  487.8   
    19    	　北京理工大学 　　	  474.0   
    20    	　　南开大学 　　　	  465.3<jupyter_text>正则表达式<jupyter_code># regular expression, regex, RE
import re
match = re.match(r'[1-9]\d{5}', '165000 JGD')
if match:
    print(match.group(0))
ls = re.findall(r'[1-9]\d{5}', 'BIT2019209 JGD165000')
ls
re.split(r'[1-9]\d{5}', 'BIT201209 JGD165000')
re.split(r'[1-9]\d{5}', 'BIT201209 JGD165000', maxsplit=1)
for m in re.finditer(r'[1-9]\d{5}', 'BIT201209 JGD165000'):
    if m:
        print(m.group(0))
re.sub(r'[1-9]\d{5}', ' ZIPCODE', 'BIT201209 JGD165000')<jupyter_output><empty_output><jupyter_text>面向对象用法：多次使用<jupyter_code>pat = re.compile(r'[1-9]\d{5}')
rst = pat.search('BIT 165000 JDJ 100302')
rst.group(0)
rst.string
rst.re
rst.pos
rst.endpos
rst.start()
rst.end()
rst.span()
match = re.search(r'PY.*?N', 'PYANBNCNDN')
match.group(0)<jupyter_output><empty_output><jupyter_text>淘宝商品比价<jupyter_code>import requests
import re
def getHTMLText(url):
    try:
        r = requests.get(url, headers = {'user-agent':'Mozilla/5.0'}, timeout = 30)
        r.raise_for_status
        r.encoding = r.apparent_encoding
        print('爬虫成功')
        return r.text
    
    except:
        print( "ERRPR" )
        
def parsePage(ilt, html):
    try:
        plt = re.findall(r'\"\_blank\" title\=\"\"[\d\.]*"', html)
        tlt = re.findall(r'\"\_blank\" title\=\" \".*?"', html)
        for i in range(len(plt)):
            price = eval(plt[i].split(':')[1])
            title = eval(tlt[i].split(':')[1])
            ilt.append([price, title])
    
    except:
        print("ParsePage Error")
    
def printGoodsList(ilt):
    try:
        tplt = "{:4}\t{:8}\t{:16}"
        print(tplt.format('序号','价格','商品名称'))
        count = 0
        for g in ilt:
            count = count + 1
            print(tplt.format(count, g[0], g[1]))
    except:
        print("")
    
def main():
    goods = 'iphone'
    depth = 3
    start_url = 'https://search.jd.com/Search?keyword=' + goods
    infoList = []
    
    for i in range(depth):
        try:
            url = start_url + '&page=' + str(i) + '&s=' + str(31*i)
            html = getHTMLText(url)
            parsePage(infoList, html)
            
            
        except:
            continue #print('main Error')
            
    printGoodsList(infoList)
    
main()<jupyter_output>爬虫成功
爬虫成功
爬虫成功
序号  	价格      	商品名称<jupyter_text>由于淘宝现在查询价格需要登陆账号，遂这里使用京东进行比较。京东不同于淘宝，价格、商品名称隐藏在tag中，可能需要美味汤进行初步处理，再用re。<jupyter_code>goods = 'iphone'
depth = 1
start_url = 'https://search.jd.com/Search?keyword=' + goods
infoList = []

for i in range(depth):
    try:
        url = start_url + '&page=' + str(i) + '&s=' + str(31*i)
        html = getHTMLText(url)
        parsePage(infoList, html)
    except:
        print('error')
html<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15765199161425578, 'fraction_numerical': 0.04654088050314465, 'mean_word_length': 2.5373208106772123, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 7, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '411148', 'license': 'MIT', 'n_tokens_mistral': 3301, 'n_tokens_neox': 3181, 'n_words': 467, 'path': 'Crawler/CRMOOC.ipynb', 'repo_name': 'JP-Xu/Protein-Aptamer-Docking', 'size': '7155'}