<commit_before># coding=utf-8

# Python modules
import re
from bs4 import BeautifulSoup

# Local modules
from common import debug, text_utils, web_utils, constants

HTML_HEADER_TAGS = ["h1", "h2", "h3", "h4", "h5", "h6"]
HTML_TEXT_TAGS = ["p"]

HTML_ITEM_TAG = "a"
HTML_LINK_TAG = "href"


# Tags
def html_common_tags():
    tags = []
    tags.extend(HTML_HEADER_TAGS)
    tags.extend(HTML_TEXT_TAGS)

    return tags


def soupify_tags(tags):
    return ",".join(tags)


def html_p_tag():
    return ",".join(HTML_TEXT_TAGS)


def extract_html(html, top=None, bottom=None):
    if top is None or bottom is None:
        return html

    start = html.find(top)
    if start == -1:
        return html

    end = html.find(bottom, start)
    return html[start:end]


def fetch_html(url, start=None, end=None):
    result = web_utils.fetch_url(url)

    html = extract_html(result.content, start, end)

    return html

<commit_msg>Reading in web_utils rather than html_utils<commit_after># coding=utf-8

# Python modules
import re
from bs4 import BeautifulSoup

# Local modules
from common import debug, text_utils, web_utils, constants

HTML_HEADER_TAGS = ["h1", "h2", "h3", "h4", "h5", "h6"]
HTML_TEXT_TAGS = ["p"]

HTML_ITEM_TAG = "a"
HTML_LINK_TAG = "href"


# Tags
def html_common_tags():
    tags = []
    tags.extend(HTML_HEADER_TAGS)
    tags.extend(HTML_TEXT_TAGS)

    return tags


def soupify_tags(tags):
    return ",".join(tags)


def html_p_tag():
    return ",".join(HTML_TEXT_TAGS)


def get_url(html):
    return html.URL


def extract_html(html, top=None, bottom=None):
    if top is None or bottom is None:
        return html

    start = html.find(top)
    if start == -1:
        return html

    end = html.find(bottom, start)
    return html[start:end]


def fetch_html(url, start=None, end=None):
    result = web_utils.fetch_url(url)

    html = extract_html(result, start, end)

    return html


{'fraction_non_alphanumeric': 0.12012480499219969, 'fraction_numerical': 0.0093603744149766, 'mean_word_length': 3.580952380952381, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1161318', 'n_tokens_mistral': 713, 'n_tokens_neox': 687, 'n_words': 181}