<commit_before># -*- coding: utf-8 -*-
import os.path
import logging
import time
import json
from data_interfaces import hansard


class FetchMPs:
    def __init__(self, **kwargs):
        self._logger = logging.getLogger("spud")
        self.hansard = hansard.TWFYHansard()
        # local directory to save fetched files to
        self.STORE_DIR = "store"
        # get the current path
        self.current_path = os.path.dirname(os.path.abspath(__file__))

    def run(self):
        self._logger.info("Fetching MPs")
        mp_ids = self.get_overview_data()
        for mp_id in mp_ids:
            if os.path.exists(os.path.join(self.current_path, self.STORE_DIR, "%s.json" % mp_id)):
                continue
            self.get_mp_info(mp_id)

    def get_overview_data(self):
        self._logger.info("Fetching MP data from TheyWorkForYou")
        mps = self.hansard.get_mps()
        time.sleep(0.5)
        path = os.path.join(self.current_path, self.STORE_DIR, "mps_overview.json")
        with open(path, "w") as f:
            json.dump(mps, f)
        return [mp["person_id"] for mp in mps]

    def get_mp_info(self, mp_id):
        extra_fields = ", ".join(["wikipedia_url", "bbc_profile_url", "date_of_birth", "mp_website", "guardian_mp_summary", "journa_list_link"])
        info = self.hansard.get_mp_info(mp_id, fields=extra_fields)
        time.sleep(0.5)

        info["details"] = self.hansard.get_mp_details(mp_id)
        time.sleep(0.5)

        path = os.path.join(self.current_path, self.STORE_DIR, "%s.json" % mp_id)
        with open(path, "w") as f:
            json.dump(info, f)

def fetch(**kwargs):
    FetchMPs(**kwargs).run()<commit_msg>Bring MP scraper inline with current standards<commit_after># -*- coding: utf-8 -*-
from datetime import datetime
import os.path
import logging
import time
import json
from data_interfaces import hansard
from utils import mongo


class FetchMPs:
    def __init__(self, **kwargs):
        self._logger = logging.getLogger("spud")
        self.hansard = hansard.TWFYHansard()
        # local directory to save fetched files to
        self.STORE_DIR = "store"
        # get the current path
        self.current_path = os.path.dirname(os.path.abspath(__file__))
        # database stuff
        self.db = mongo.MongoInterface()
        self.COLLECTION_NAME = "mps_fetch"
        if kwargs["refreshdb"]:
            self.db.drop(self.COLLECTION_NAME)
        # if True, avoid downloading where possible
        self.dryrun = kwargs["dryrun"]

    def run(self):
        self._logger.info("Fetching MPs")
        mp_ids = self.get_overview_data()
        for mp_id in mp_ids:
            self.get_mp_info(mp_id)

    def get_overview_data(self):
        self._logger.info("Fetching MP data from TheyWorkForYou")
        mps = self.hansard.get_mps()
        time.sleep(0.5)
        path = os.path.join(self.current_path, self.STORE_DIR, "mps_overview.json")
        with open(path, "w") as f:
            json.dump(mps, f)
        return [mp["person_id"] for mp in mps]

    def get_mp_info(self, mp_id):
        fetched = False
        filename = os.path.join(self.STORE_DIR, "%s.json" % mp_id)

        if not self.dryrun:
            extra_fields = ", ".join(["wikipedia_url", "bbc_profile_url", "date_of_birth", "mp_website", "guardian_mp_summary", "journa_list_link"])
            info = self.hansard.get_mp_info(mp_id, fields=extra_fields)
            time.sleep(0.5)

            info["details"] = self.hansard.get_mp_details(mp_id)
            fetched = str(datetime.now())
            time.sleep(0.5)

            path = os.path.join(self.current_path, filename)
            with open(path, "w") as f:
                json.dump(info, f)

        meta = {
            "filename": filename,
            "source": {
                "url": "http://www.theyworkforyou.com/api/docs/getMP?id=%s#output" % mp_id,
                "linked_from_url": None,
                "fetched": fetched,
            }
        }
        self.db.save(self.COLLECTION_NAME, meta)

def fetch(**kwargs):
    FetchMPs(**kwargs).run()
{'fraction_non_alphanumeric': 0.123991195891416, 'fraction_numerical': 0.0034238200048911714, 'mean_word_length': 2.422594142259414, 'pattern_counts': {'":': 5, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '782287', 'n_tokens_mistral': 1394, 'n_tokens_neox': 1361, 'n_words': 276}