<commit_before>
# coding=utf-8

# Python modules
import urllib
from bs4 import BeautifulSoup

# Google App Engine Modules
from google.appengine.api import urlfetch, urlfetch_errors

# Local modules
from common import debug, html_utils, text_utils, constants
from common.telegram import telegram_utils

# Link to fetch html from
DG_URL = "https://www.desiringgod.org/articles"

# Which class to isolate?
DG_SELECT = "share share--card js-share-values"

def fetch_desiringgod(query=""):
    formatUrl = DG_URL
    if(text_utils.is_valid(query)):
        formatUrl.append("/" + query)

    html = html_utils.fetch_html(formatUrl)
    if html is None:
        return None

    debug.log("Html: " + html)

    soup = html_utils.html_to_soup(html.main)

    return soup 

def get_desiringgod_raw(query=""):
    soup = fetch_desiringgod(query)
    if soup is None:
        return None

    blocks = []
    for tag in soup.select(DG_SELECT):
        blocks.append({"text":tag["data-title"], "url":tag["data-link"]})

    return blocks

def get_desiringgod(query=""):
    passage = get_desiringgod_raw(query)

    if passage is None:
        return None

    return passage<commit_msg>Updating string<commit_after>
# coding=utf-8

# Python modules
import urllib
from bs4 import BeautifulSoup

# Google App Engine Modules
from google.appengine.api import urlfetch, urlfetch_errors

# Local modules
from common import debug, html_utils, text_utils, constants
from common.telegram import telegram_utils

# Link to fetch html from
DG_URL = "https://www.desiringgod.org/articles"

# Which class to isolate?
DG_SELECT = "share share--card js-share-values"

def fetch_desiringgod(query=""):
    formatUrl = DG_URL
    if(text_utils.is_valid(query)):
        formatUrl = formatUrl + "/" + query

    html = html_utils.fetch_html(formatUrl)
    if html is None:
        return None

    debug.log("Html: " + html)

    soup = html_utils.html_to_soup(html.main)

    return soup 

def get_desiringgod_raw(query=""):
    soup = fetch_desiringgod(query)
    if soup is None:
        return None

    blocks = []
    for tag in soup.select(DG_SELECT):
        blocks.append({"text":tag["data-title"], "url":tag["data-link"]})

    return blocks

def get_desiringgod(query=""):
    passage = get_desiringgod_raw(query)

    if passage is None:
        return None

    return passage
{'fraction_non_alphanumeric': 0.10211864406779661, 'fraction_numerical': 0.001694915254237288, 'mean_word_length': 3.5491329479768785, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '99557', 'n_tokens_mistral': 774, 'n_tokens_neox': 747, 'n_words': 199}