# Path: common_utils.py
# def fetch_html_page(page_url):
#     user_agent = (
#         "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) "
#         "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.89 Safari/537.36"
#     )
#     headers = {"User-Agent": user_agent}
#     page = requests.get(page_url, headers=headers, timeout=10)
#     return page.text
# 
# def html_parser_from(page_html):
#     return BeautifulSoup(page_html, "html.parser")


import logging
import os
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from datetime import datetime
from pathlib import Path
from subprocess import CalledProcessError
from jinja2 import Environment, FileSystemLoader
from py_executable_checklist.workflow import (
    WorkflowBase,
    notify_me,
    run_command,
    run_workflow,
)
from slug import slug
from common_utils import fetch_html_page, html_parser_from
        known_domains = ["ycombinator", "algolia", "hackernews", "youtube", "twitter", "chrome.google.com", "youtu.be"]

        def has_known_domain(post_link):
            return any(map(lambda l: l in post_link.lower(), known_domains))

        return link.startswith("http") and not has_known_domain(link)

    def run(self, context):
        valid_links = [
            link
            for link in self.all_links
            if self.is_valid_link(link) and self.accessible(link, self.child_links_folder)
        ]

        # output
        context["valid_links"] = valid_links


class GrabChildLinkTitle(WorkflowBase):
    """Get page title for each valid link"""

    valid_links: list
    child_links_folder: Path

    def page_title_from(self, child_links_folder, link_in_comment):
        page_slug = slug(link_in_comment)
        page_path = f"{page_slug}.html"

        post_html_page_file = child_links_folder / page_path
        page_html = fetch_html(link_in_comment, post_html_page_file)

# Path: common_utils.py
# def fetch_html_page(page_url):
#     user_agent = (
#         "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) "
#         "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.89 Safari/537.36"
#     )
#     headers = {"User-Agent": user_agent}
#     page = requests.get(page_url, headers=headers, timeout=10)
#     return page.text
# 
# def html_parser_from(page_html):
#     return BeautifulSoup(page_html, "html.parser")


# Path: links_to_hugo.py
import logging
import os
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from datetime import datetime
from pathlib import Path
from subprocess import CalledProcessError
from jinja2 import Environment, FileSystemLoader
from py_executable_checklist.workflow import (
    WorkflowBase,
    notify_me,
    run_command,
    run_workflow,
)
from slug import slug
from common_utils import fetch_html_page, html_parser_from

        known_domains = ["ycombinator", "algolia", "hackernews", "youtube", "twitter", "chrome.google.com", "youtu.be"]

        def has_known_domain(post_link):
            return any(map(lambda l: l in post_link.lower(), known_domains))

        return link.startswith("http") and not has_known_domain(link)

    def run(self, context):
        valid_links = [
            link
            for link in self.all_links
            if self.is_valid_link(link) and self.accessible(link, self.child_links_folder)
        ]

        # output
        context["valid_links"] = valid_links


class GrabChildLinkTitle(WorkflowBase):
    """Get page title for each valid link"""

    valid_links: list
    child_links_folder: Path

    def page_title_from(self, child_links_folder, link_in_comment):
        page_slug = slug(link_in_comment)
        page_path = f"{page_slug}.html"

        post_html_page_file = child_links_folder / page_path
        page_html = fetch_html(link_in_comment, post_html_page_file)

        bs = html_parser_from(page_html)