<jupyter_start><jupyter_text>給定文章標題，爬取 PTT 上的所有相關文章- title: [新聞] 2噸水晶球沿街滾 撞壞5輛汽機車和民宅- URL encoing (UTF-8)- combine URL path<jupyter_code>import requests
import re
import json

from bs4 import BeautifulSoup, NavigableString
from pprint import pprint
from urllib.parse import urlencode, urljoin
QUERY_TITLE = '[新聞] 2噸水晶球沿街滾 撞壞5輛汽機車和民宅'
cookies = {'over18': '1'}<jupyter_output><empty_output><jupyter_text>URL encoding取得相同文章標題的列表<jupyter_code>encoding_title = urlencode({'q': QUERY_TITLE})
query = 'https://www.ptt.cc/bbs/Gossiping/search?{}'.format(encoding_title)
print(query)
resp_article_list = requests.get(query, cookies=cookies)
soup_article_list = BeautifulSoup(resp_article_list.text, 'lxml')<jupyter_output><empty_output><jupyter_text>列出所有文章並爬取<jupyter_code>def crawl_article(url):
    resp = requests.get(url, cookies={'over18': '1'})
    if resp.status_code != 200:
        return
    soup = BeautifulSoup(resp.text, 'lxml')
    print('Start to Crawling', url)

    # ##############################
    # crawl article
    # ##############################
    article = {
        'author_id': '',
        'author_nickname': '',
        'title': '',
        'timestamp': '',
        'contents': '',
        'ip': ''
    }
    article_body = soup.find(id='main-content')

    # article header
    article_head = article_body.findAll('div', class_='article-metaline')
    for metaline in article_head:
        meta_tag = metaline.find(class_='article-meta-tag').text
        meta_value = metaline.find(class_='article-meta-value').text
        if meta_tag == '作者':
            compile_nickname = re.compile('\((.*)\)').search(meta_value)
            article['author_id'] = meta_value.split('(')[0].strip(' ')
            article['author_nickname'] = compile_nickname.group(1) if compile_nickname else ''
        elif meta_tag == '標題':
            article['title'] = meta_value
        elif meta_tag == '時間':
            article['timestamp'] = meta_value

    #  article content
    contents = [expr for expr in article_body.contents if isinstance(expr, NavigableString)]
    contents = [re.sub('\n', '', expr) for expr in contents]
    contents = [i for i in contents if i]
    contents = '\n'.join(contents)
    article['contents'] = contents

    # article publish ip
    article_ip = article_body.find(class_='f2').text
    compile_ip = re.compile('[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}').search(article_ip)
    article['ip'] = compile_ip.group(0) if compile_ip else ''

    # ##############################
    # crawl comments
    # ##############################
    comments = []
    for comment in article_body.findAll('div', class_='push'):
        tag = comment.find(class_='push-tag').text
        guest_id = comment.find(class_='push-userid').text
        guest_content = comment.find(class_='push-content').text
        guest_ipdatetime = comment.find(class_='push-ipdatetime').text
        compile_ip = re.compile('[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}').search(guest_ipdatetime)
        guest_ip = compile_ip.group(0) if compile_ip else ''
        guest_timestamp = re.sub(guest_ip, '', guest_ipdatetime).strip()
        comments.append({
            'tag': tag,
            'id': guest_id,
            'content': guest_content,
            'ip': guest_ip,
            'timestamp': guest_timestamp
        })
    
    article['comments'] = comments
    article['url'] = url
    return article
data = []
for article_line in soup_article_list.findAll('div', class_='r-ent'):
    title_tag = article_line.find('div', class_='title')
    article_url = title_tag.find('a')['href']
    article_url = urljoin(resp_article_list.url, article_url)
    article_data = crawl_article(article_url)
    data.append(article_data)

with open('search_api_by_title.json', 'w+', encoding='utf-8') as f:
    json.dump(data, f, indent=2, ensure_ascii=False)
    print('Save - search_api_by_title.json')<jupyter_output>Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537874850.A.20D.html
Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537868945.A.8A9.html
Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537861382.A.154.html
Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537859788.A.BE2.html
Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537859045.A.287.html
Save - search_api_by_title.json
{'fraction_non_alphanumeric': 0.17645703754895187, 'fraction_numerical': 0.026491591799124626, 'mean_word_length': 3.6538049303322615, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '420029', 'license': 'Apache-2.0', 'n_tokens_mistral': 1622, 'n_tokens_neox': 1575, 'n_words': 248, 'path': 'appendix_ptt/01_search_api_by_title.ipynb', 'repo_name': 'afunTW/Python-Crawling-Tutorial', 'size': '4341'}