<commit_before>#!/usr/bin/env python3

import os
import requests
import re
import arrow
from requests.adapters import HTTPAdapter
from bs4 import BeautifulSoup
import functools


datadir = os.path.expanduser("~/.scp-data/")
req = requests.Session()
req.mount("http://www.scp-wiki.net/", HTTPAdapter(max_retries=5))


class cache_to_disk():
    """Save the results of the func to the directory specified in datadir."""

    def __init__(self, func):
        self.path = os.path.join(datadir, func.__name__)
        self.func = func
        if not os.path.exists(self.path):
            os.makedirs(self.path)
        functools.update_wrapper(self, func)

    def __call__(self, *args, **kwargs):
        filename = self._make_filename(*args, **kwargs)
        fullpath = os.path.join(self.path, filename)
        if os.path.isfile(fullpath):
            with open(fullpath) as cached_file:
                data = cached_file.read()
        else:
            print("{:<40}{}".format(self.func.__name__, filename))
            data = self.func(*args, **kwargs)
            if data is not None:
                with open(fullpath, "w") as cached_file:
                    cached_file.write(data)
        return data

    def __get__(self, obj, objtype):
        '''Support instance methods.'''
        return functools.partial(self.__call__, obj)

    def _make_filename(self, *args, **kwargs):
        #this WILL NOT WORK in a general case
        #it does just enough for what we need and nothing more
        if args == () and kwargs == {}:
            return "no_args"
        url = getattr(args[0], "url", None)
        if url is not None:
            replace = ["http://", "/", "www.scp-wiki.net", ":"]
            for i in (url.replace(r, "") for r in replace):
                url = i
            return url
        return repr(args[0])


def wikidot_module(name, page, perpage, pageid):
    """Retrieve data from the specified wikidot module."""
    headers = {"Content-Type": "application/x-www-form-urlencoded;",
               "Cookie": "wikidot_token7=<PASSWORD>;"}
    payload = {"page": page, "perpage": perpage, "page_id": pageid,
               "moduleName": name, "wikidot_token7": "<PASSWORD>"}
    return req.post("http://www.scp-wiki.net/ajax-module-connector.php",
                    data=payload, headers=headers).json()["body"]


def _init_titles():
        """Return a dict of SCP articles' titles"""
        index_urls = ["http://www.scp-wiki.net/scp-series",
                      "http://www.scp-wiki.net/scp-series-2",
                      "http://www.scp-wiki.net/scp-series-3"]
        titles = {}
        for url in index_urls:
            soup = BeautifulSoup(req.get(url).text)
            articles = [i for i in soup.select("ul > li")
                        if re.search("(SCP|SPC)-[0-9]+", i.text)]
            for e in articles:
                k, v = e.text.split(" - ", maxsplit=1)
                titles[k] = v
        return titles


def _init_images():
    url = "http://scpsandbox2.wikidot.com/ebook-image-whitelist"
    soup = BeautifulSoup(req.get(url).text)
    results = {}
    for i in soup.select("tr")[1:]:
        results[i.select("td")[0].text] = i.select("td")[1].text
    return results


def _init_rewrites():
    url = "http://05command.wikidot.com/alexandra-rewrite"
    soup = BeautifulSoup(req.get(url).text)
    results = {}
    pref = "http://www.scp-wiki.net/"
    for i in soup.select("tr")[1:]:
        results[pref + i.select("td")[0].text] = i.select("td")[1].text
    return results

titles = _init_titles()
images = _init_images()
rewrites = _init_rewrites()


class Page():

    """Scrape and store contents and metadata of a page."""

    def __init__(self, url=None):
        self.url = url
        self.data = None
        self.title = None
        self.tags = []
        self.images = []
        self.authors = []
        self.history = None
        self.soup = None<commit_msg>use pickle for cache files; cache more functions<commit_after>#!/usr/bin/env python3

from bs4 import BeautifulSoup
import arrow
import functools
import os
import pickle
import re
import requests


datadir = os.path.expanduser("~/.scp-data/")
req = requests.Session()
req.mount("http://www.scp-wiki.net/",
          requests.adapters.HTTPAdapter(max_retries=5))


class cache_to_disk():
    """Save the results of the func to the directory specified in datadir."""

    def __init__(self, func):
        self.path = os.path.join(datadir, func.__name__)
        self.func = func
        if not os.path.exists(self.path):
            os.makedirs(self.path)
        functools.update_wrapper(self, func)

    def __call__(self, *args, **kwargs):
        filename = self._make_filename(*args, **kwargs)
        fullpath = os.path.join(self.path, filename)
        if os.path.isfile(fullpath):
            with open(fullpath, "rb") as cache_file:
                data = pickle.load(cache_file)
        else:
            print("{:<40}{}".format(self.func.__name__, filename))
            data = self.func(*args, **kwargs)
            if data is not None:
                with open(fullpath, "wb") as cache_file:
                    pickle.dump(data, cache_file)
        return data

    def __get__(self, obj, objtype):
        '''Support instance methods.'''
        return functools.partial(self.__call__, obj)

    def _make_filename(self, *args, **kwargs):
        #this WILL NOT WORK in a general case
        #it does just enough for what we need and nothing more
        if args == () and kwargs == {}:
            return "no_args"
        url = getattr(args[0], "url", None)
        if url is not None:
            replace = ["http://", "/", "www.scp-wiki.net", ":"]
            for i in (url.replace(r, "") for r in replace):
                url = i
            return url
        return repr(args[0])


def wikidot_module(name, page, perpage, pageid):
    """Retrieve data from the specified wikidot module."""
    headers = {"Content-Type": "application/x-www-form-urlencoded;",
               "Cookie": "wikidot_token7=123456;"}
    payload = {"page": page, "perpage": perpage, "page_id": pageid,
               "moduleName": name, "wikidot_token7": "<PASSWORD>"}
    return req.post("http://www.scp-wiki.net/ajax-module-connector.php",
                    data=payload, headers=headers).json()["body"]


@cache_to_disk
def _init_titles():
    """Return a dict of SCP articles' titles"""
    index_urls = ["http://www.scp-wiki.net/scp-series",
                  "http://www.scp-wiki.net/scp-series-2",
                  "http://www.scp-wiki.net/scp-series-3"]
    titles = {}
    for url in index_urls:
        soup = BeautifulSoup(req.get(url).text)
        articles = [i for i in soup.select("ul > li")
                    if re.search("(SCP|SPC)-[0-9]+", i.text)]
        for e in articles:
            k, v = e.text.split(" - ", maxsplit=1)
            titles[k] = v
    return titles


@cache_to_disk
def _init_images():
    url = "http://scpsandbox2.wikidot.com/ebook-image-whitelist"
    soup = BeautifulSoup(req.get(url).text)
    results = {}
    for i in soup.select("tr")[1:]:
        results[i.select("td")[0].text] = i.select("td")[1].text
    return results


@cache_to_disk
def _init_rewrites():
    url = "http://05command.wikidot.com/alexandra-rewrite"
    soup = BeautifulSoup(req.get(url).text)
    results = {}
    pref = "http://www.scp-wiki.net/"
    for i in soup.select("tr")[1:]:
        results[pref + i.select("td")[0].text] = i.select("td")[1].text
    return results


titles = _init_titles()
images = _init_images()
rewrites = _init_rewrites()


class Page():

    """Scrape and store contents and metadata of a page."""

    def __init__(self, url=None):
        self.url = url
        self.data = None
        self.title = None
        self.tags = []
        self.images = []
        self.authors = []
        self.history = None
        self.soup = None
{'fraction_non_alphanumeric': 0.1467069638584561, 'fraction_numerical': 0.006548293665785166, 'mean_word_length': 2.2858916011584607, 'pattern_counts': {'":': 16, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 16, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4175738', 'n_tokens_mistral': 2592, 'n_tokens_neox': 2532, 'n_words': 601}