<jupyter_start><jupyter_text>練習- 觀察 https://afuntw.github.io/Test-Crawling-Website/pages/gallery/index.html 並撰寫爬蟲程式- 判斷是否為 JavaScript rendered website- 下載網頁影片- 設定 Implicit Wait- 透過 XPath 定位圖片**透過靜態網站爬蟲會看到的圖片是****透過動態網站爬蟲會看到的圖片是**<jupyter_code>import os
import requests
import re

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from fake_useragent import UserAgent
from pprint import pprint

url = 'https://afuntw.github.io/Test-Crawling-Website/pages/gallery/index.html'
fu = UserAgent()<jupyter_output><empty_output><jupyter_text>使用 requests 做靜態爬蟲<jupyter_code>resp = requests.get(url)
soup = BeautifulSoup(resp.text, 'lxml')
imgs = soup.find_all('img', class_=re.compile('.*img-change'))
imgs = [i['src'] for i in imgs]
imgs = list(set(imgs))
pprint(imgs)<jupyter_output>['https://i.imgur.com/0s6Iiu3.png']<jupyter_text>使用 Selenium 做動態爬蟲<jupyter_code>driver = webdriver.Chrome()
results = os.path.abspath('../results')
if not os.path.exists(results):
    os.makedirs(results)

try:
    # webdriver setting
    driver.get(url)
    driver.maximize_window()
    driver.implicitly_wait(10)
    
    # xpath
    imgs = driver.find_elements(By.XPATH, '/html/body/div/div/div/a/img')
    imgs = [i.get_attribute('src') for i in imgs]
    imgs = list(set(imgs))
    print(imgs)
    
    # download
    for img in imgs:
        headers = {'User-Agent': fu.random}
        img_resp = requests.get(img, stream=True, headers=headers)
        
        filename = os.path.basename(img)
        print('catch - {}'.format(filename))
        filename = os.path.join(results, filename)
        
        with open(filename, 'wb') as f:
            for chunk in img_resp.iter_content(2048):
                f.write(chunk)
            print('save - {}'.format(filename))
    
except Exception as e:
    print(e)
finally:
    driver.quit()<jupyter_output>['https://i.imgur.com/db3tGBG.png']
catch - db3tGBG.png
save - /home/dirl/github/Python-Crawling-Tutorial/results/db3tGBG.png
{'fraction_non_alphanumeric': 0.13676907829534193, 'fraction_numerical': 0.006442021803766105, 'mean_word_length': 3.6844547563805103, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '39711', 'license': 'Apache-2.0', 'n_tokens_mistral': 830, 'n_tokens_neox': 832, 'n_words': 132, 'path': '02_selenium/00_selenium_crawling_render_image.ipynb', 'repo_name': 'afunTW/Python-Crawling-Tutorial', 'size': '2018'}