<commit_before># -*- coding: utf-8 -*-
import logging
import os.path
import requests
from bs4 import BeautifulSoup
from utils import mongo

class ScrapeAPPC:
    def __init__(self):
        self._logger = logging.getLogger('spud')
        # local directory to save fetched files to
        self.STORE_DIR = "store"
        # get the current path
        self.current_path = os.path.dirname(os.path.abspath(__file__))
        self.db = mongo.MongoInterface()
        self.PREFIX = "appc"

    def run(self):
        metas = self.db.fetch_all("%s_fetch" % self.PREFIX, paged=False)
        for meta in metas:
            if meta["description"] == "current":
                agency = self.scrape_current(meta)
                self.db.save("%s_scrape" % self.PREFIX, agency)
            else:
                # TODO: Scrape PDFs
                # agencies = self.scrape_pdf(meta)
                pass

    def scrape_current(self, meta):
        full_path = os.path.join(self.current_path, self.STORE_DIR, meta["date_to"], meta["filename"])
        with open(full_path) as f:
            html = f.read()
        soup = BeautifulSoup(html).find(class_="member-profile")

        name = [x for x in soup.find("h1").stripped_strings][0]

        address_soups = soup.find(class_="profile-address").find_all("tr")[1:]
        addresses = []
        contacts = []
        for address_soup in address_soups:
            address, contact = [[x for x in y.stripped_strings] for y in address_soup.find_all("td")]
            if address != []:
                addresses.append(address)
            if contact != []:
                contacts.append(contact)

        country_soup = soup.find(class_="profile-country")
        countries = [x.text.strip() for x in country_soup.find_all("li")] if country_soup else []

        staff_soup = soup.find(class_="profile-staff")
        staff = [x.text.strip() for x in staff_soup.find_all("li")] if staff_soup else []

        clients = {"pro-bono": None, "consultancy": None, "monitoring": None}
        client_soups = soup.find_all(class_="profile-clients")
        for client_soup in client_soups:
            client_table_heading = client_soup.find("th").text
            if "Pro-Bono Clients" in client_table_heading:
                client_type = "pro-bono"
            elif "UK PA consultancy" in client_table_heading:
                client_type = "consultancy"
            elif "UK monitoring" in client_table_heading:
                client_type = "monitoring"
            else:
                raise Exception("Unknown client type: '%s'" % client_table_heading)
            clients[client_type] = []
            for client in client_soup.find_all("li"):<commit_msg>Split staff list into `has_pass` and `no_pass`<commit_after># -*- coding: utf-8 -*-
import logging
import os.path
import requests
from bs4 import BeautifulSoup
from utils import mongo

class ScrapeAPPC:
    def __init__(self):
        self._logger = logging.getLogger('spud')
        # local directory to save fetched files to
        self.STORE_DIR = "store"
        # get the current path
        self.current_path = os.path.dirname(os.path.abspath(__file__))
        self.db = mongo.MongoInterface()
        self.PREFIX = "appc"

    def run(self):
        metas = self.db.fetch_all("%s_fetch" % self.PREFIX, paged=False)
        for meta in metas:
            if meta["description"] == "current":
                agency = self.scrape_current(meta)
                self.db.save("%s_scrape" % self.PREFIX, agency)
            else:
                # TODO: Scrape PDFs
                # agencies = self.scrape_pdf(meta)
                pass

    def scrape_current(self, meta):
        full_path = os.path.join(self.current_path, self.STORE_DIR, meta["date_to"], meta["filename"])
        with open(full_path) as f:
            html = f.read()
        soup = BeautifulSoup(html).find(class_="member-profile")

        name = [x for x in soup.find("h1").stripped_strings][0]

        address_soups = soup.find(class_="profile-address").find_all("tr")[1:]
        addresses = []
        contacts = []
        for address_soup in address_soups:
            address, contact = [[x for x in y.stripped_strings] for y in address_soup.find_all("td")]
            if address != []:
                addresses.append(address)
            if contact != []:
                contacts.append(contact)

        country_soup = soup.find(class_="profile-country")
        countries = [x.text.strip() for x in country_soup.find_all("li")] if country_soup else []

        staff_soup = soup.find(class_="profile-staff")
        all_staff = [x.text.strip() for x in staff_soup.find_all("li")] if staff_soup else []
        staff = {"has_pass": [], "no_pass": []}
        for staff_name in all_staff:
            if staff_name.endswith(" *"):
                staff["has_pass"].append(staff_name[:-2].strip())
            else:
                staff["no_pass"].append(staff_name)

        clients = {"pro-bono": [], "consultancy": [], "monitoring": []}
        client_soups = soup.find_all(class_="profile-clients")
        for client_soup in client_soups:
            client_table_heading = client_soup.find("th").text
            if "Pro-Bono Clients" in client_table_heading:
                client_type = "pro-bono"
            elif "UK PA consultancy" in client_table_heading:
                client_type = "consultancy"
            elif "UK monitoring" in client_table_heading:
                client_type = "monitoring"
            else:
                raise Exception("Unknown client type: '%s'" % client_table_heading)
            for client in client_soup.find_all("li"):
{'fraction_non_alphanumeric': 0.11913166254853512, 'fraction_numerical': 0.001941404871161313, 'mean_word_length': 2.2364363221016563, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3685161', 'n_tokens_mistral': 1729, 'n_tokens_neox': 1684, 'n_words': 383}