<jupyter_start><jupyter_text>Scraping Hong Kong Government websites for Tender Requests on Digital ProjectsList of government websites to scrape:  1. https://www.gld.gov.hk/egazette/english/search_gazette/search_gov_notice.html2. https://www.cedb.gov.hk/ccib/eng/tender/index.htm3. https://www.edb.gov.hk/en/public-admin/tender-notices/tender-notice/index.html4. https://www.itb.gov.hk/en/tender/5. https://www.itc.gov.hk/en/tender/tender.htm6. http://www.aud.gov.hk/eng/tender/tender.htm7. http://www.ams.gov.hk/eng/tender1.htm8. https://www.admwing.gov.hk/eng/tender/tender.htm9. http://www.cas.gov.hk/eng/notice/notice_tender.html10. https://www.cad.gov.hk/english/tender.html11. https://www.csb.gov.hk/english/tender/17.html12. https://www.cedb.gov.hk/citb/en/Tender_Notices/index.html13. https://www.cr.gov.hk/en/tender/index.htm14. http://www.cmab.gov.hk/en/tender/index.htm15. https://www.csd.gov.hk/english/tender/other_tender.html16. https://www.customs.gov.hk/en/customs_notices/tender/index.html17. http://www.dh.gov.hk/english/tender/tender.html18. https://www.doj.gov.hk/eng/tender/19. http://www.effo.gov.hk/en/news-events/tender-notice.html20. https://www.enb.gov.hk/en/susdev/tender/index.htm21. https://www.fstb.gov.hk/tb/en/tender-exercises.htm22. https://www.fstb.gov.hk/fsb/tender/index.htm23. http://www.hkfsd.gov.hk/eng/tender/tender_notice/index.html24. http://www.fhb.gov.hk/en/tender/25. http://www.gfs.gov.hk/eng/tender.htm26. http://www.hkma.gov.hk/eng/about-the-hkma/tender-invitations.shtml27. http://www.hko.gov.hk/tender/tender.htm28. https://www.police.gov.hk/ppp_en/11_useful_info/tenders.html29. https://www.hongkongpost.hk/en/about_us/tender/notice/index.html30. http://www.icac.org.hk/en/so/tender/index.html31. https://www.ird.gov.hk/eng/ten/ten.htm32. http://www.try.gov.hk/internet/ehtender.html33. https://www.tid.gov.hk/english/aboutus/noticeboard/tender.html34. http://www.tourism.go (broken? - not included)35. http://www.sb.gov.hk/eng/tender/index.htmv.hk/english/other/other_tender.html (broken - not included)<jupyter_code>from bs4 import BeautifulSoup
import requests
import pandas as pd
import numpy as np

# import sys # import 
# !{sys.executable} -m pip install oauth2client<jupyter_output><empty_output><jupyter_text>Detect if website has tenders<jupyter_code>gov_list = ['https://www.gld.gov.hk/egazette/english/search_gazette/search_gov_notice.html',
'https://www.cedb.gov.hk/ccib/eng/tender/index.htm',
'https://www.edb.gov.hk/en/public-admin/tender-notices/tender-notice/index.html',
'https://www.itb.gov.hk/en/tender/',
'https://www.itc.gov.hk/en/tender/tender.htm',
'http://www.aud.gov.hk/eng/tender/tender.htm',
'http://www.ams.gov.hk/eng/tender1.htm',
'https://www.admwing.gov.hk/eng/tender/tender.htm',
'http://www.cas.gov.hk/eng/notice/notice_tender.html',
'https://www.cad.gov.hk/english/tender.html',
'https://www.csb.gov.hk/english/tender/17.html',
'https://www.cedb.gov.hk/citb/en/Tender_Notices/index.html',
'https://www.cr.gov.hk/en/tender/index.htm',
'http://www.cmab.gov.hk/en/tender/index.htm',
# 'https://www.csd.gov.hk/english/tender/other_tender.html', really long title
'https://www.customs.gov.hk/en/customs_notices/tender/index.html',
'http://www.dh.gov.hk/english/tender/tender.html',
'https://www.doj.gov.hk/eng/tender/',
'http://www.effo.gov.hk/en/news-events/tender-notice.html',
'https://www.enb.gov.hk/en/susdev/tender/index.htm',
'https://www.fstb.gov.hk/tb/en/tender-exercises.htm',
'https://www.fstb.gov.hk/fsb/tender/index.htm',
'http://www.hkfsd.gov.hk/eng/tender/tender_notice/index.html',
'http://www.fhb.gov.hk/en/tender/',
'http://www.gfs.gov.hk/eng/tender.htm',
# 'http://www.hkma.gov.hk/eng/about-the-hkma/tender-invitations.shtml', tender names do not have standard words
'http://www.hko.gov.hk/tender/tender.htm',
'https://www.police.gov.hk/ppp_en/11_useful_info/tenders.html',
'https://www.hongkongpost.hk/en/about_us/tender/notice/index.html',
'http://www.icac.org.hk/en/so/tender/index.html',
'https://www.ird.gov.hk/eng/ten/ten.htm',
'http://www.try.gov.hk/internet/ehtender.html',
'https://www.tid.gov.hk/english/aboutus/noticeboard/tender.html',]
link_list = []
status_list = []
tender_name_list = []

for link in gov_list:
    page = requests.get(link)
    status = page.status_code
    
    
    if status == 200:
        soup = BeautifulSoup(page.content, 'html.parser')
        sentences = soup.text.split('\n')
        sentences = list(filter(None, sentences))
        sentences = list(filter(lambda space: space.strip(), sentences))
        
        for sentence in sentences:
            if ('Provision' in sentence and 'Provisions' not in sentence) or 'Supply' in sentence or 'Quotation' in sentence:
                link_list.append(link)
                status_list.append('Received')
                tender_name_list.append(sentence)
csd_url = 'https://www.csd.gov.hk/english/tender/other_tender.html'
page = requests.get(csd_url)
soup = BeautifulSoup(page.content,'html.parser')
tables = soup.find_all('tbody')[1:3]

for table in tables:
    sentences = table.get_text().split('\t')
    sentences = list(filter(None, sentences))
    sentences = list(filter(lambda name: name.strip(), sentences))
    
    for sentence in sentences:
        if ('Provision' in sentence and 'Provisions' not in sentence) or 'Supply' in sentence or 'Quotation' in sentence:
            link_list.append(csd_url)
            status_list.append('Received')
            tender_name_list.append(sentence)
hkma_url = 'http://www.hkma.gov.hk/eng/about-the-hkma/tender-invitations.shtml'
page = requests.get(hkma_url)
soup = BeautifulSoup(page.content,'html.parser')
item = soup.find('div', class_='item')
tenders = item.find_all('a')
for tender in tenders:
    link_list.append(hkma_url)
    status_list.append('Received')
    tender_name_list.append("Provision of " + tender.get_text())
d = {'links': link_list,
     'status': status_list,
     'tender_name': tender_name_list,
    }

df = pd.DataFrame(data=d)
df = df[['links', 'status', 'tender_name']]
def clean_name(row):
    name_parts = row.tender_name.split(' ')
    for i in name_parts:
        if i == 'Provision' or i == 'Supply' or i == 'Quotation':
            start = name_parts.index(i)
    clean = name_parts[start:]
    together = ' '.join(clean)
    return together
df['tender_name'] = df.apply(clean_name, axis=1)
digital_terms = ['software', 'digital', 'comput', 'system', 'mobile', 
                 'web', 'consult', 'cms', 'network', 'infrastructure']

def digital(row):
    for term in digital_terms:
        if term in row.tender_name.lower():
            return True
    return False

exact_digital_terms = ['app', 'apps', 'it', 'erp']

def exact_words(row):
    name = row.tender_name.lower()
    parts = name.split()
    for term in exact_digital_terms:
        for part in parts:
            if term == part:
                return True
    return row.digital
df['digital'] = df.apply(digital, axis=1)
df['digital'] = df.apply(exact_words, axis=1)
df
for i in range(0, len(df)):
    print(df['tender_name'][i])
df.loc[df['digital'] == True]
import pygsheets
import datetime
"""
the following code writes down the tender in the following google sheet
https://drive.google.com/open?id=1akjLfewescBZ6NUF_7hAQOafZgDRos6uBhbYKm2zHVU
"""
gc = pygsheets.authorize(outh_file='client_secret_161825456738-6qpud9vevgsdjq6oudagoe891nqs40u8.apps.googleusercontent.com.json')
sh = gc.open('tender-scraping')
wks = sh.sheet1
wks.clear(start='A1')

current_time = datetime.datetime.now().strftime("Time: %H:%M:%S | Date: %d/%m/%Y")

wks.update_cell('A1', current_time)
wks.set_dataframe(df, 'A2')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1784551257001433, 'fraction_numerical': 0.014589032174026312, 'mean_word_length': 3.841109709962169, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 42, 'lorem ipsum': 0, 'www.': 70, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '259317', 'license': 'MIT', 'n_tokens_mistral': 3164, 'n_tokens_neox': 2972, 'n_words': 383, 'path': 'tender_scraping.ipynb', 'repo_name': 'jchiu1303/scraping_tenders', 'size': '7677'}