<jupyter_start><jupyter_text><jupyter_code>##################################
#
# Script to scrape PDF documents
#   from https://www.youtube.com/watch?v=VDd6dVrYzao
#
#              by
#
#       <NAME>
#
##################################

#Install libraries
%%capture
!pip install BeautifulSoup
!pip install requests
!pip install lxml
# packages
import requests
from bs4 import BeautifulSoup
from urllib.request import unquote

# target URL
url = 'https://www.health.go.ke/press-releases/'

# make HTTP GET request to the target URL
print('HTTP GET: %s', url)
response = requests.get(url)

# parse content
content = BeautifulSoup(response.text, 'lxml')

# extract URLs referencing PDF documents
all_urls = content.find_all('a')

# loop over all URLs
for url in all_urls:
    # try URLs containing 'href' attribute
    try:
        # pick up only those URLs containing 'pdf'
        # within 'href' attribute
        if 'pdf' in url['href']:
            # init PDF url
            pdf_url = ''
            
            # append base URL if no 'https' available in URL
            if 'https' not in url['href']:
                pdf_url = 'https://www.health.go.ke/press-releases/' + url['href']

            # otherwise use bare URL
            else:
                pdf_url = url['href']
                pdf = []
            
            # make HTTP GET request to fetch PDF bytes
            print('HTTP GET: %s', pdf_url)          
            pdf_response = requests.get(pdf_url)

            
            # extract  PDF file name
            filename = unquote(pdf_response.url).split('/')[-1].replace(' ', '_')
            
            # write PDF to local file
            with open('/content/data_pdfs/' + filename, 'wb') as f:
            #with open('/content/data.pdf', 'wb') as f:
                # write PDF to local file
                f.write(pdf_response.content)
    
    # skip all the other URLs
    except:
        pass
# Install pyPDF package
%%capture
!pip install PyPDF2

# Import libraries
import re
import PyPDF2
import pandas as pd
def extract_county_cases(text):
  
  #For counties with 2 names, change it to one name.
  text = re.sub(r'\band\b','',text) 
  text = re.sub(r'[()]?[,]?[.]?\.?','',text) 
  text = text.replace("Homa Bay", "HomaBay")
  text = text.replace("West Pokot","WestPokot")
  text = text.replace("Tharaka Nithi ","TharakaNithi ")
  text = text.replace("Murang’a","Muranga")
  #Tana 
  text = text.replace("Tana River","TanaRiver")
  #Taita/Taveta
  text = text.replace("Taita Taveta","TaitaTaveta")
  #Trans Nzoia
  text = text.replace("Trans Nzoia","TransNzoia")
  #Uasin Gishu
  text = text.replace("Uasin Gishu","UasinGishu")
  #Elgeyo/Marakwet
  text = text.replace("Elgeyo Marakwet","ElgeyoMarakwet")
  per_county21= text.split()
  #print(per_county21)
  
  county_data = pd.DataFrame()
  county_data['Names'] = per_county21
  county_data['count'] = county_data['Names']

  #county_data.loc[county_data.Names == county_data.count, "count"] = ''
  # df.iloc[::2]
  just_county = county_data.iloc[::2].reset_index()
  just_county.drop('index',axis = 1, inplace= True)
  just_county
  vals = county_data.iloc[1::2].reset_index()
  just_county['count'] = vals['count']
  return just_county
# remove parenthesis
def parenthesis(text):
  import re
  text=text.strip()
  #text = re.sub(r'Distribution of positive cases by age is as follows; ', '', text)
  text = re.sub(r'[a-zA-Z\s;]', '',text)
  text=re.sub(r'[\n]+', ' ',text)
  text=re.sub(r'[\s]+', '',text)
  text = re.sub(r'[()\s.]',' ',text)
  text = re.sub(r'[,]','',text)
  text = re.sub(r'\\n\s+',' ',text)
  text = re.sub(r'years',' ',text)
  text = re.sub(r'and',' ',text)
  text = re.sub(r'above',' ',text)
  return text
# importing required modules 
%%capture
import PyPDF2
import re

def clean(text):
  # text = re.sub("Murang’a",'Muranga',text)
  # text = re.sub('^\n\d[\n\s?\n?]*','',text)
  
  text = re.sub('\n*','',text)
  return text    
# creating a pdf file object 
pdfFileObj = open('/content/drive/MyDrive/Data Science/data_pdfs/Press-Statement-on-Covid-19-10th-August-2021.pdf', 'rb') 
    
# creating a pdf reader object 
pdfReader = PyPDF2.PdfFileReader(pdfFileObj) 
pdf = []   
# printing number of pages in pdf file 
print(pdfReader.numPages) 
i = 0
for i in range(0,pdfReader.numPages):    
  # creating a page object 
  pageObj = pdfReader.getPage(i) 
  i +=1   
  # extracting text from page 
  # import re
  extract = clean(pageObj.extractText())
  pdf.append(extract)
  i +=1
  # extract = extract.strip()
  # extract=re.sub(r'[\s]+', '',extract)
  print(pageObj.extractText())
  # print(extract)
  full_text = ' '.join(pdf)  
# closing the pdf file object 
pdfFileObj.close()

# County distribution
def county_distribution(extracted_text):
  start = extracted_text.find('In terms of County distribution; ') + 33
  end = extracted_text.find('.', start)
  new_cases_percounty = extracted_text[start:end]
  # new_cases_percounty = parenthesis(new_cases_percounty)
  new_cases_percounty = extract_county_cases(new_cases_percounty)
  new_cases_percounty.rename(columns={'Names': 'Age_group', 'count': 'New_cases_percounty'}, inplace = True)
  return new_cases_percounty
  # return extracted_text[start:end]

county_distribution(full_text)
# positive cases by age
def age_positive(text):
  start = text.find('Distribution of positive cases by age is as follows; ') + 53
  end = text.find('.', start)
  new_cases = text[start:end]
  new_cases = parenthesis(new_cases)
  new_cases = extract_county_cases(new_cases)
  new_cases.rename(columns={'Names': 'Age_group', 'count': 'New_cases'}, inplace = True)
  return new_cases
  # return text[start:end]

age_positive(full_text)
# Deaths by age
def deaths_age(text):
  start = text.find('New deaths by age; ') + 19
  end = text.find('.', start)
  total_deaths = text[start:end]
  total_deaths = parenthesis(total_deaths)
  total_deaths = extract_county_cases(total_deaths)
  total_deaths.rename(columns={'Names': 'Age_group', 'count': 'Fatalities'}, inplace = True)
  return total_deaths

deaths_age(full_text)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15368248772504092, 'fraction_numerical': 0.006055646481178396, 'mean_word_length': 3.4671052631578947, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '130317', 'license': 'RSA-MD', 'n_tokens_mistral': 2177, 'n_tokens_neox': 2108, 'n_words': 486, 'path': 'New_Cases_and_Deaths_Scraping.ipynb', 'repo_name': 'MaureenGatu/omdena-kenya-covid-analysis', 'size': '6120'}