<jupyter_start><jupyter_text>---- Functions and Settings for HOMM webscrapper----  1. Setup<jupyter_code># Based on article
# https://jakevdp.github.io/blog/2017/12/05/installing-python-packages-from-jupyter/
"""
!pip install requests>=2.23.0 --quiet --yes --user
!pip install beautifulsoup4==4.7.1 --quiet --yes --user
#!pip install pandas --quiet --yes --user 
#!pip install numpy --quiet --yes --user
#!pip install matplotlib --quiet --yes --user
#!pip install requests==2.21.0 --quiet --user 
!pip install fake-useragent==0.1.11 --quiet --yes  --user 
!pip install google==3.0.0 --quiet --yes --user
!pip install openpyxl --quiet --yes --user
!pip install xlrd --quiet --yes --user
"""
"""
# Install a pip package in the current Jupyter kernel
import sys
print("-------------------------------------\nInstalling - All requirements")
# pip
!{sys.executable} -m pip install requests>=2.23.0
!{sys.executable} -m pip install beautifulsoup4==4.7.1
!{sys.executable} -m pip install fake-useragent==0.1.11
!{sys.executable} -m pip install google==3.0.0
!{sys.executable} -m pip install openpyxl
!{sys.executable} -m pip install xlrd
# conda
#!conda install --yes --prefix {sys.prefix} numpy
#print("==> DONE - installing.")
#print("-------------------------------------\nImporting - All requirements")
"""

"""
# Install a pip package in the current Jupyter kernel
import sys
print("-------------------------------------\nInstalling - All requirements")
# pip

!python -m pip install numpy
!python -m pip install pandas
!python -m pip install matplotlib

!python -m pip install requests>=2.23.0
!python -m pip install beautifulsoup4==4.7.1
!python -m pip install fake-useragent==0.1.11
!python -m pip install google==3.0.0
!python -m pip install openpyxl
!python -m pip install xlrd
# conda
#!conda install --yes --prefix {sys.prefix} numpy

print("==> DONE - installing.")
print("-------------------------------------\nImporting - All requirements")
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches

import os
import sys
import re
import string
import traceback

from random import randint
import time
import datetime as dt
import datetime
import requests
import json

from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from itertools import cycle

import warnings
warnings.filterwarnings('ignore')
warnings.simplefilter("ignore")
print("==> DONE - importing.\n----------------------------------------")

try: 
    from googlesearch import search 
except ImportError:  
    print("No module named 'google' found") 
    
%load_ext autoreload
%autoreload 2<jupyter_output>==> DONE - importing.
----------------------------------------
The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>2. Set Globals - Folders and Files<jupyter_code># folders
CURRENT_DIR = os.curdir; print(CURRENT_DIR)
RESULTS_ARCHIVE = os.path.join("..", "results", "archive"); print(RESULTS_ARCHIVE)
RESULTS_LATEST = os.path.join("..", "results", "latest"); print(RESULTS_LATEST)
SOURCES = os.path.join("..", "sources"); print(SOURCES)
dirs = [CURRENT_DIR, RESULTS_ARCHIVE, RESULTS_LATEST, SOURCES]

for fn in dirs:
    if not os.path.exists(fn): 
        print("Making folder: ", fn)
        os.mkdir(fn)
    else:
        print("Folder already exists: ", fn)

# search terms
search_terms = "SearchTerms_MasterList.csv"
df_searchterms = pd.read_csv(os.path.join(SOURCES, search_terms))
print("Search terms df shape: ", df_searchterms.shape)<jupyter_output>.
..\results\archive
..\results\latest
..\sources
Folder already exists:  .
Folder already exists:  ..\results\archive
Folder already exists:  ..\results\latest
Folder already exists:  ..\sources
Search terms df shape:  (13, 3)<jupyter_text>3. Set Globals - Config and Inputs<jupyter_code># Set globals for Google Search
global dict_googledfs_lastmonth, dict_googledfs_alltime
dict_googledfs_lastmonth = {}
dict_googledfs_alltime = {}

# Programmables Search Engine settings/config
global dict_hvddfs_alltime, dict_ngodfs_alltime, dict_igodfs_alltime, dict_psedfs_alltime
dict_hvddfs_alltime = {}
dict_ngodfs_alltime = {}
dict_igodfs_alltime = {}
dict_psedfs_alltime = {} # This is for the custom search engine, may not be used going forward

# ReliefWeb and United Nations settings/config
global dict_rwdfs_alltime, dict_undfs_alltime
dict_rwdfs_alltime = {}
dict_undfs_alltime = {}

all_searchterms = ['"'+q+'"' + ' ' +'"'+appendix+'"' for q, appendix 
                   in zip(df_searchterms.product_title, df_searchterms.append_to_term)]
print(all_searchterms)<jupyter_output><empty_output><jupyter_text>4. Utility Functions   *Proxies and Requests*- https://httpbin.org/ip  : gives you back your ip addres- 'https://www.sslproxies.org/': gives you proxies- library `fake_user_agent` gives you headers<jupyter_code>def get_IPaddress(proxies = None): 
    url = "https://httpbin.org/ip"
    
    if proxies == None:
        print(proxies)
        res = requests.get(url)
        print(res.text)
    else:
        print(proxies)
        res = requests.get(url, proxies)
        print(res.text)
    return res
ua = UserAgent(); ua<jupyter_output><empty_output><jupyter_text>*Task complete - signal*<jupyter_code>def complete_signal(task):
    t = dt.datetime.now()
    dd = str(t.year).zfill(2)+"/"+str(t.month).zfill(2)+"/"+str(t.day).zfill(2)
    tt= str(t.hour).zfill(2)+":"+str(t.minute).zfill(2)+":"+str(t.second).zfill(2)
    tstamp = dd+"   "+tt
    # build a rectangle in axes coords
    left, width = .25, .5
    bottom, height = .25, .5
    right = left + width
    top = bottom + height

    fig = plt.figure()
    ax = fig.add_axes([0,0,1,1])
    
    ax.text(0.8*(left+right), 0.8*(bottom+top), "\n"+task+" - COMPLETE!\n\n---------\n Time: "+tstamp,
            horizontalalignment='center',
            verticalalignment='center',
            fontsize=40, color='black',
            backgroundcolor="lightgreen",
            transform=ax.transAxes)
    
    ax.set_axis_off()
    plt.show()
    return

complete_signal(task = "Running on Binder!")<jupyter_output><empty_output><jupyter_text>*Combine and Save Function*<jupyter_code>def singleitem_and_save(df, source, title, time_window):
    today = dt.date.today(); datename = str(today.year)+str(today.month).zfill(2)+str(today.day).zfill(2)
    df.to_excel(os.path.join(RESULTS_ARCHIVE,source+"_"+title+"_"+time_window+"_"+datename+".xlsx"),
                index=False)
    df.to_excel(os.path.join(RESULTS_LATEST, source+"_"+title+"_"+time_window+"_Latest.xlsx"),
                index=False)
    print("Completed scraping and file saved for: ", source, " --> ", title)
    return 

def singleitem_and_save_v2(df, source, title, time_window):
    today = dt.date.today(); datename = str(today.year)+str(today.month).zfill(2)+str(today.day).zfill(2)
    df.to_excel(os.path.join(RESULTS_ARCHIVE,source+"_"+title+"_"+time_window+"_"+datename+".xlsx"),
                index=False)
    df.to_excel(os.path.join(RESULTS_LATEST, source+"_"+title+"_"+time_window+"_Latest.xlsx"),
                index=False)
    print("Completed scraping and file saved for: ", source, " --> ", title)
    return 

def combine_and_save(df_list, source, time_window):
    combined = pd.concat(df_list);
    today = dt.date.today(); datename = str(today.year)+str(today.month).zfill(2)+str(today.day).zfill(2)
    combined.to_excel(os.path.join(RESULTS_ARCHIVE,source+"_"+time_window+"_"+datename+".xlsx"), index=False)
    combined.to_excel(os.path.join(RESULTS_LATEST,source+"_"+time_window+"_Latest.xlsx"), index=False)
    return complete_signal("Scraping "+ source +" "+time_window)

singleitem_and_save_v2(pd.DataFrame(all_searchterms), "Here", "Binded", "Now")<jupyter_output><empty_output><jupyter_text>*Get Reference info from Humanitarian Outcomes*<jupyter_code># get publication titles for HO website
def get_publication_titles(url_src):
    #url_src = "https://www.humanitarianoutcomes.org/publications"
    # go to HO website url
    #r = requests.get(url_src)
    #soup = BeautifulSoup(r.text, "html.parser")

    # scrap for titles
    with open(os.path.join(SOURCES, "20200825_publications.txt"), "r") as f:    
        txt = f.read()

    soup = BeautifulSoup(txt, "html.parser")
    
    # return as list
    titles = [h.text for h in [d.find_all("h2") for d in sp.find_all("div", class_ = "content")][0]]
    exc = 'Subscribe to our mailing list', 'Filter Publications'
    
    u_titles = []
    for t in titles:
        if t not in exc and t not in u_titles:
            u_titles.append(t)
    print("number of pubs: ", len(u_titles))
    
    return u_titles 

publications = get_publication_titles()
print(publications)
BeautifulSoup
search<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16395679363274587, 'fraction_numerical': 0.012052302444570779, 'mean_word_length': 3.5788651743883393, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 36, 'https://': 5, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '569019', 'license': 'MIT', 'n_tokens_mistral': 3095, 'n_tokens_neox': 2944, 'n_words': 692, 'path': 'code/funcs.ipynb', 'repo_name': 'tichmangono/binder_deploy_test', 'size': '8795'}