<jupyter_start><jupyter_text>練習- 觀察 https://www.thsrc.com.tw/tw/TimeTable/SearchResult 並撰寫爬蟲程式- 抓取一個禮拜後的高鐵時刻表- 台北到台南下午兩點的班次- 使用 requests + BeautifulSoup 實作- 透過 pandas 輸出成 csv<jupyter_code>import os
import requests
import pandas as pd
import datetime

from bs4 import BeautifulSoup

url = 'https://www.thsrc.com.tw/tw/TimeTable/SearchResult'
after_one_week = datetime.datetime.now() + datetime.timedelta(weeks=1)
after_one_week_format = after_one_week.strftime('%Y/%m/%d')
print('The date after one week - {}'.format(after_one_week_format))

form_data = {
    'StartStation': '977abb69-413a-4ccf-a109-0272c24fd490',
    'EndStation': '9c5ac6ca-ec89-48f8-aab0-41b738cb1814',
    'SearchDate': after_one_week_format,
    'SearchTime': '14:00',
    'SearchWay': 'DepartureInMandarin',
    'RestTime': '',
    'EarlyOrLater': ''
}
resp = requests.post(url, data=form_data)
resp.encoding = 'utf-8'
soup = BeautifulSoup(resp.text, 'lxml')
rows = soup.table.find_all('tr', recursive=False)

colname, rows = rows[1], rows[2:]
colname = list(colname.stripped_strings)

for i, row in enumerate(rows):
    trips = row.find('td', class_='column1')
    t_departure = row.find('td', class_='column3')
    t_arrive = row.find('td', class_='column4')
    duration = row.find('td', class_='column2')
    early_ticket = row.find('td', class_='Width1')
    
    trips = trips.text if trips else None
    t_departure = t_departure.text if t_departure else ''
    t_arrive = t_arrive.text if t_arrive else ''
    duration = duration.text if duration else ''
    early_ticket = list(early_ticket.stripped_strings) if early_ticket else ''
    early_ticket = early_ticket[0] if early_ticket else ''
    
    rows[i] = [trips, t_departure, t_arrive, duration, early_ticket]

df = pd.DataFrame(rows, columns=colname)
df
results = os.path.abspath('../results')
if not os.path.exists(results):
    os.makedirs(results)

filename = os.path.join(results, 'thsrc_{}.csv'.format(after_one_week.strftime('%Y%m%d')))
df.to_csv(filename, index=False)
print('Save csv to {}'.format(filename))<jupyter_output>Save csv to /home/afun/github/Python-Crawling-Tutorial/results/thsrc_20180228.csv
{'fraction_non_alphanumeric': 0.1454460966542751, 'fraction_numerical': 0.028810408921933085, 'mean_word_length': 4.947513812154696, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '73320', 'license': 'Apache-2.0', 'n_tokens_mistral': 902, 'n_tokens_neox': 857, 'n_words': 149, 'path': '00_GET_POST/01_POST_thsrc_time_table.ipynb', 'repo_name': 'afunTW/Python-Crawling-Tutorial', 'size': '2152'}