<jupyter_start><jupyter_text>South china morning post-Web scraping<jupyter_code>! pip install beautifulsoup4<jupyter_output>Collecting beautifulsoup3<jupyter_text>Import libraries<jupyter_code>from urllib.request import urlopen
from bs4 import BeautifulSoup
import csv
import numpy as np<jupyter_output><empty_output><jupyter_text>Load the URL and store it<jupyter_code>#Open URL and read it
url = urlopen("http://www.scmp.com/content/search/canada?f[0]=im_field_label_tag%3A544&f[1]=ds_created%3A%5B2008-03-01T00%3A00%3A00Z%20TO%202017-02-29T23%3A59%3A59Z%5D")
content = url.read()
soup = BeautifulSoup(content)
# The are 12 results webpages with 10 news articles per webpage.
links=[] #create a list of links to the 5 pages
wbp="http://www.scmp.com"
for divTag in soup.find_all( "div", {"class" : "item-list"}):
    for ulTag in divTag.find_all( "li", {"class" : "pager-item"}):
        links.append(wbp+ulTag.a["href"])<jupyter_output><empty_output><jupyter_text>Extract data from each hyperlink and store it in the dictionary<jupyter_code>#Initialize empty dictionary to fill with data 
my_dict={}
wbp="http://www.scmp.com"
count_1=0
for link in links: 
    count_1=count_1+1
    #Open link
    url = urlopen(link) #Open webpage of results
    content = url.read()
    soup = BeautifulSoup(content)
    count_2=0
    for divTag in soup.findAll("div", { "class" : "search-result-wrapper"}): #Get the links of the pages of news
        count_2=count_2+1
        page=wbp+divTag.h3.a['href'] 
        try:
            print('Link:',count_1,'Page:',count_2)
            print(page)
            url = urlopen(page)  #Open page of the news article
            content = url.read()
            soup = BeautifulSoup(content)
            content_temp=''
            my_dict['Page-'+str((count_1-1)*10+count_2)]={}
            for divTag_2 in soup.findAll("h1", { "class" : "title"}):   #Get headline
                my_dict['Page-'+str((count_1-1)*10+count_2)]['headline']=divTag_2.text
            for divTag_3 in soup.findAll("div", { "class" : "pane-content"}): #Get content
                for divTag_4 in divTag_3.findAll("p"):
                    try:
                        content_temp=content_temp+' '+divTag_4.text
                    except:
                        continue
            my_dict['Page-'+str((count_1-1)*10+count_2)]['content']= content_temp
        except:
            continue
#encode weird characters to ascii
my_dict_1={}
import unicodedata
for x in range(1,121):
    my_dict_1['Page-'+str(x)]={}
    my_dict_1['Page-'+str(x)]['content']=unicodedata.normalize('NFKD',my_dict['Page-'+str(x)]['content']).encode('ascii','ignore')
#encode content to utf-8
my_dict_fin={}
for x in range(1,121):
    my_dict_fin['Page-'+str(x)]={}
    my_dict_fin['Page-'+str(x)]['content'] = str(my_dict_1['Page-'+str(x)]['content'], "utf-8")
#Write CSV file
with open("scmp_news.csv","w") as toWrite:
    writer=csv.writer(toWrite,delimiter=",")
    writer.writerow([ "Headline","Content"])
    for x in range(1,121):
        x=str(x)
        writer.writerow([my_dict['Page-'+x]['headline'], my_dict_fin['Page-'+x]['content']])<jupyter_output><empty_output><jupyter_text>Import CSV File and open<jupyter_code>#Import CSV file to open
import pandas
news_articles = pandas.read_csv('scmp_news.csv', encoding = "ISO-8859-1")
news_articles<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15040047463660636, 'fraction_numerical': 0.03292791456541086, 'mean_word_length': 3.8378766140602583, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '410198', 'license': 'MIT', 'n_tokens_mistral': 1254, 'n_tokens_neox': 1198, 'n_words': 234, 'path': 'South_china_webscrape.ipynb', 'repo_name': 'jomavera/Work', 'size': '3371'}