<commit_before>"""Creates a PDF from an article of the project „<NAME> Nicht-Freaks“."""

import time
import os
import shelve
import sys
import logging
import requests

report_logger = logging.getLogger("report_logger")
report_logger.setLevel(logging.DEBUG)
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO)
report_logger.addHandler(console_handler)

try:
    os.mkdir("out")
except FileExistsError:
    pass

file_handler = logging.FileHandler(os.path.join("out", "parser_log.log"), mode="w")
file_handler.setLevel(logging.DEBUG)
report_logger.addHandler(file_handler)

from mfnf.api import HTTPMediaWikiAPI
from mfnf.parser import ArticleParser
from mfnf.utils import CachedFunction
from mfnf.sitemap import parse_sitemap
from mfnf.latex import LatexExporter, MediaWiki2Latex
from mfnf.utils import to_snake_case, open_database

# title of article which shall be converted to PDF
SITEMAP_ARTICLE_NAME = "<NAME> Nicht-Freaks: Projekte/LMU Buchprojekte"
CACHE = ".cache/cache.db"

def create_book(book, api):
    """Creates the LaTeX file of a book."""
    book_name = to_snake_case(book["name"])
    target = os.path.join("out", book_name, book_name + ".tex")

    try:
        os.makedirs(os.path.dirname(target))
    except FileExistsError:
        pass

    report_logger.info("= Export of Book {} =".format(book["name"]))
    book = MediaWiki2Latex()(book)

    with open(target, "w") as latex_file:
        LatexExporter(api, os.path.dirname(target))(book, latex_file)

def run_script():
    """Runs this script."""
    with open_database(CACHE) as database:
        cached_function = CachedFunction(database)

        class CachedMediaWikiAPI(HTTPMediaWikiAPI):
            """A version of the API where the main function calls are
            cached."""

            @cached_function
            def convert_text_to_html(self, title, text):
                return super().convert_text_to_html(title, text)

            @cached_function
            def normalize_formula(self, formula, mode):
                return super().normalize_formula(formula, mode)

            @cached_function
            def get_revisions(self, title):
                # TODO: The list of revisions can change, thus this caching
                # need to be implemented in a better way in the future.
                return super().get_revisions(title)

        ses = requests.Session()
        ses.mount("", requests.adapters.HTTPAdapter(max_retries=5))

        api = CachedMediaWikiAPI(ses)<commit_msg>create_books: Use urllib3.util.retry.Retry<commit_after>"""Creates a PDF from an article of the project „Mathe für Nicht-Freaks“."""

import time
import os
import shelve
import sys
import logging

report_logger = logging.getLogger("report_logger")
report_logger.setLevel(logging.DEBUG)
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO)
report_logger.addHandler(console_handler)

try:
    os.mkdir("out")
except FileExistsError:
    pass

file_handler = logging.FileHandler(os.path.join("out", "parser_log.log"), mode="w")
file_handler.setLevel(logging.DEBUG)
report_logger.addHandler(file_handler)

import requests
from urllib3.util.retry import Retry

from mfnf.api import HTTPMediaWikiAPI
from mfnf.parser import ArticleParser
from mfnf.utils import CachedFunction
from mfnf.sitemap import parse_sitemap
from mfnf.latex import LatexExporter, MediaWiki2Latex
from mfnf.utils import to_snake_case, open_database

# title of article which shall be converted to PDF
SITEMAP_ARTICLE_NAME = "Mathe für Nicht-Freaks: Projekte/LMU Buchprojekte"
CACHE = ".cache/cache.db"

def create_book(book, api):
    """Creates the LaTeX file of a book."""
    book_name = to_snake_case(book["name"])
    target = os.path.join("out", book_name, book_name + ".tex")

    try:
        os.makedirs(os.path.dirname(target))
    except FileExistsError:
        pass

    report_logger.info("= Export of Book {} =".format(book["name"]))
    book = MediaWiki2Latex()(book)

    with open(target, "w") as latex_file:
        LatexExporter(api, os.path.dirname(target))(book, latex_file)

def run_script():
    """Runs this script."""
    with open_database(CACHE) as database:
        cached_function = CachedFunction(database)

        class CachedMediaWikiAPI(HTTPMediaWikiAPI):
            """A version of the API where the main function calls are
            cached."""

            @cached_function
            def convert_text_to_html(self, title, text):
                return super().convert_text_to_html(title, text)

            @cached_function
            def normalize_formula(self, formula, mode):
                return super().normalize_formula(formula, mode)

            @cached_function
            def get_revisions(self, title):
                # TODO: The list of revisions can change, thus this caching
                # need to be implemented in a better way in the future.
                return super().get_revisions(title)

        ses = requests.Session()
        retry = Retry(total=10, connect=5, read=5, backoff_factor=1)
        ses.mount("", requests.adapters.HTTPAdapter(max_retries=retry))

        api = CachedMediaWikiAPI(ses)
{'fraction_non_alphanumeric': 0.09907478797224364, 'fraction_numerical': 0.002313030069390902, 'mean_word_length': 3.4926406926406925, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '57497', 'n_tokens_mistral': 1613, 'n_tokens_neox': 1557, 'n_words': 400}