<jupyter_start><jupyter_text>文字處理函數<jupyter_code># 這個函式所有商品都可以用
# 回傳的東西 (售價，售量)
def get_the_dealinfo(htmlscript):
    try:
    # 拿到售價
    # 雖然select出來裡面只有一個東西但因為出來是一個list，要把它拿來用還是要加[0]把內容取出來
    # find(text=True, recursive=False) 這個用法會只抓標籤最外層的東西，標籤裡面就不拿了
    # 
        deal_price = int(float(
        htmlscript.parent.parent.select('.productPrice')[0].find('em').find(text=True, recursive=False)))
    # 拿到 銷售量
        get_dealnumber_with_chinese =htmlscript.parent.parent.select('.productStatus')[0].find('em').text
        print(get_dealnumber_with_chinese)
    # 處理中文
    # 如果拿到的字理面有萬，把後面兩個字"萬"去掉
        if re.search("萬", get_dealnumber_with_chinese):
            deal_number = int(float(get_dealnumber_with_chinese[:-1])*10000)
        # 其餘狀況
        else:
            deal_number = int(get_dealnumber_with_chinese)

        return deal_price,deal_number

# 如果搜尋的頁面找不到對應的商家
    except:
            return 0,0

# 轉換中文到網址的格式
def chinese_to_url(chinese):
    url = urllib.parse.quote(chinese)
    return url 

# 試看看有沒有ban掉
# sort = d 放在gcp上面會被ban掉
# 試看看能不能放在GCP
url = 'https://list.tmall.com/search_shopitem.htm?user_id=883737303&q=nex3&sort=td'
source = requests.get(url,headers = headers).text
soup = Bt4(source,'html.parser')
# print(soup)
# ,headers = headers

ban_by_Tmall = soup.find("p").text

# 如果IP被Tmall ban掉的話
if '二正忙' in ban_by_Tmall:
	print("目前被ban了")
	
else:
#     TODO
    # 店家的部分想要改成使用 user_id=883737303 的方式來把 商家的名字 去做對應
    # key:user_id , value:商家名字
    shop_official = {
        '883737303':'Vivo官方旗艦店',
        '2838892713':'華為官方旗艦店',
        '1714128138':'小米官方旗艦店',
        '901409638':'Oppo官方旗艦店',
        '370627083':'三星官方旗艦店'}
    shop_dealer = {'2616970884':'蘇寧易購官方旗艦店','268451883':'三際數碼官方旗艦店'}
    shop_dealer_list = ['蘇寧易購官方旗艦店','三際數碼官方旗艦店']
    
    # 合併兩個 dictionary
    shop_all = {** shop_official, **shop_dealer}
    # 把key跟value對調
    shop_all_R = {value: key for key, value in shop_all.items()}

    #搜尋p30 = 搜尋p30+pro
    product_of_url = {'小米9':chinese_to_url('小米9'),
                    'Opporeno':'oppo+reno',
                    'Vivo-Nex3':'vivo+nex3',
                    'Vivo-Nex3-5G':'nex3',
                    'Vivo-X27':'vivo+X27',
                    '華為-P30':'p30',
                    '華為-Mate30':'mate30',
                    '華為-Mate30Pro':'mate30',
                    '三星-Note10':'note10',
                    '三星-Note10-5G':'note10',
                     } 

#     '小米9':'小米官方旗艦店'
    shop_product_id = {
    'Vivo官方旗艦店':{'Vivo-Nex3-5G':'601903805660','Vivo-Nex3':'601717764536'
#                  ,'Vivo-X27':'590812353617'
                },
    '華為官方旗艦店':{
#         '華為-P30':'','華為-P30Pro':'',
               '華為-Mate30':'602918373522','華為-Mate30Pro':'603330883901'
#              ,'華為-Mate30-5G':'','華為-Mate30Pro-5G':''
    },
#     '小米官方旗艦店':['小米9':'','小米9-5G':''],
#     'Oppo官方旗艦店':['Oppo-Reno':''],
    '三星官方旗艦店':{
                '三星-Note10':'600469192419','三星-Note10-5G':'601093283698'
#                ,'Note10plus':'','Note10plus-5G':''
    },
        
    '蘇寧易購官方旗艦店':{
                'Vivo-Nex3-5G':'600891657529','Vivo-Nex3':'602577338304'
#                  ,'Vivo-X27':'',
                ,'華為-Mate30':'600662536849'
                ,'三星-Note10':'600453072145','三星-Note10-5G':'600453936466'
                },
    
    '三際數碼官方旗艦店':{
                '華為-Mate30':'603187202582','華為-Mate30Pro':'602973425447'
                ,'三星-Note10':'600932362045','三星-Note10-5G':'600933006447'
    }}
    

    products_of_shops = {'小米9':[shop_official['1714128138']],
                    'Opporeno':[shop_official['901409638']],
                    'Vivo-Nex3':[shop_official['883737303']],
                    '華為Mate30-Pro':[shop_official['2838892713']]}
    regular_expression_of_product = {'小米9':'小米9',
                            'Opporeno':'reno',
                            'Vivo-Nex3':'nex3',
                            '華為Mate30-Pro':'mate30pro'}
# 把非官方的網站跟官方的網站合起來,這裡 sum(欲合併的陣列,[])會把所有的陣列攤平合在一起,並做 flatten array
#     products_of_shops 的 key 是所有的產品
#     products_of_shops 的 value 是所有產品相關的店家
# 這裡稍微注意華為官方網站 的商品有兩個
    for key in products_of_shops.keys():
        products_of_shops[key] = sum([products_of_shops[key],shop_dealer_list],[])
    

    shop_id = {}
    for shop_item in shop_product_id.keys(): #countries.keys 是全部的型號
        shop_id[shop_item] = [v for v in shop_product_id[shop_item].values()]
        
    product_id = {}
    for product_item in shop_product_id.values():
        for product in product_item.keys():
            product_id.setdefault(product,[]) # added key
            product_id[product] += [product_item[product]]<jupyter_output><empty_output><jupyter_text>爬蟲主體<jupyter_code>res = []
d = {}
#迴圈跑每個產品
# product:'mate30','nex30'...
for shop in shop_product_id.keys():
    for product in shop_product_id[shop].keys():

        url = f'https://list.tmall.com/search_shopitem.htm?user_id={shop_all_R[shop]}\
        &start_price=1000&q={product_of_url[product]}&sort=td'
        print(url)
        sessions = requests.session()
        sessions.headers = headers
        source = requests.get(url,headers = headers).text
        soup = Bt4(source,'html.parser')
        # search_id 是一個正則表示 要拿去對 html 搜尋 <a>標籤裡的 href 有沒有符合的
        search_id = re.compile(shop_product_id[shop][product])
        deal_price,deal_number = get_the_dealinfo(soup.find('a',href = search_id))
        if deal_price == 0:
            print('被ban')
        else:
            d['Product'] = product
            d['Shopname'] = shop
            d['Dealprice'] = deal_price
            d['Dealnumber'] = deal_number
            d['Timestamp'] = datetime.datetime.today().strftime("%Y-%m-%d")
            res.append(d)
            print(d)
            d = {}
#         等一會再跑下一圈
        time.sleep(random.randint(1,3))
print('爬蟲結束')
# df = pd.DataFrame(res)<jupyter_output>https://list.tmall.com/search_shopitem.htm?user_id=883737303        &start_price=1000&q=nex3&sort=td
4550
{'Product': 'Vivo-Nex3-5G', 'Shopname': 'Vivo官方旗艦店', 'Dealprice': 5698, 'Dealnumber': 4550, 'Timestamp': '2019-10-21'}
https://list.tmall.com/search_shopitem.htm?user_id=883737303        &start_price=1000&q=vivo+nex3&sort=td
678
{'Product': 'Vivo-Nex3', 'Shopname': 'Vivo官方旗艦店', 'Dealprice': 4998, 'Dealnumber': 678, 'Timestamp': '2019-10-21'}
https://list.tmall.com/search_shopitem.htm?user_id=2838892713        &start_price=1000&q=mate30&sort=td
2.7萬
{'Product': '華為-Mate30', 'Shopname': '華為官方旗艦店', 'Dealprice': 4299, 'Dealnumber': 27000, 'Timestamp': '2019-10-21'}
https://list.tmall.com/search_shopitem.htm?user_id=2838892713        &start_price=1000&q=mate30&sort=td
4.8萬
{'Product': '華為-Mate30Pro', 'Shopname': '華為官方旗艦店', 'Dealprice': 5799, 'Dealnumber': 48000, 'Timestamp': '2019-10-21'}
https://list.tmall.com/search_shopitem.htm?user_id=370627083        &start_price=1000&q=note10&sort[...]<jupyter_text>檔案處理<jupyter_code>if platform.system() == "Windows":
    # Local 端
    path = "static/data/Tmall5g.csv"
else:
    # AWS 端
    path = "/home/ubuntu/iphone11/mainweb/static/data/Tmall5g.csv"

path = 'static/data/Tmall5g.csv'
Data = pd.read_csv(path)
Old_Data = Data.to_dict("records")
newres = res + Old_Data
df = pd.DataFrame(res)
df = df.drop_duplicates()
df.to_csv(path, encoding="utf_8_sig", index=False)<jupyter_output><empty_output><jupyter_text>測試<jupyter_code># 把url都印出來
for shop in shop_product_id.keys():
    for product in shop_product_id[shop].keys():
        url = f'https://list.tmall.com/search_shopitem.htm?user_id={shop_all_R[shop]}&start_price=1000&q={product_of_url[product]}&sort=td&style=w'
        print(url)<jupyter_output>https://list.tmall.com/search_shopitem.htm?user_id=883737303&start_price=1000&q=nex3&sort=td&style=w
https://list.tmall.com/search_shopitem.htm?user_id=883737303&start_price=1000&q=vivo+nex3&sort=td&style=w
https://list.tmall.com/search_shopitem.htm?user_id=2838892713&start_price=1000&q=mate30&sort=td&style=w
https://list.tmall.com/search_shopitem.htm?user_id=2838892713&start_price=1000&q=mate30&sort=td&style=w
https://list.tmall.com/search_shopitem.htm?user_id=370627083&start_price=1000&q=note10&sort=td&style=w
https://list.tmall.com/search_shopitem.htm?user_id=370627083&start_price=1000&q=note10&sort=td&style=w
https://list.tmall.com/search_shopitem.htm?user_id=2616970884&start_price=1000&q=nex3&sort=td&style=w
https://list.tmall.com/search_shopitem.htm?user_id=2616970884&start_price=1000&q=vivo+nex3&sort=td&style=w
https://list.tmall.com/search_shopitem.htm?user_id=2616970884&start_price=1000&q=mate30&sort=td&style=w
https://list.tmall.com/search_shopitem.htm?user_id=2616970884&star[...]
{'fraction_non_alphanumeric': 0.1606313834726091, 'fraction_numerical': 0.08762766945218199, 'mean_word_length': 3.338872104733132, 'pattern_counts': {'":': 1, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 18, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '664241', 'license': 'bzip2-1.0.6', 'n_tokens_mistral': 4291, 'n_tokens_neox': 4002, 'n_words': 354, 'path': 'Tmall.ipynb', 'repo_name': 'ti-lei/iphone11', 'size': '8616'}