<commit_before>__package__ = 'archivebox.extractors'

from pathlib import Path
from tempfile import NamedTemporaryFile

from typing import Optional
import json

from ..index.schema import Link, ArchiveResult, ArchiveError
from ..system import run, atomic_write
from ..util import (
    enforce_types,
    download_url,
    is_static_file,
    
)
from ..config import (
    TIMEOUT,
    CURL_BINARY,
    SAVE_READABILITY,
    DEPENDENCIES,
    READABILITY_VERSION,
)
from ..logging_util import TimedProgress

@enforce_types
def get_html(link: Link, path: Path) -> str:
    """
    Try to find wget, singlefile and then dom files.
    If none is found, download the url again.
    """
    canonical = link.canonical_outputs()
    abs_path = path.absolute()
    sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]]
    document = None
    for source in sources:
        try:
            with open(abs_path / source, "r") as f:
                document = f.read()
                break
        except (FileNotFoundError, TypeError):
            continue
    if document is None:
        return download_url(link.url)
    else:
        return document

@enforce_types
def should_save_readability(link: Link, out_dir: Optional[str]=None) -> bool:
    out_dir = out_dir or link.link_dir
    if is_static_file(link.url):
        return False

    output = Path(out_dir or link.link_dir) / 'readability'
    return SAVE_READABILITY and READABILITY_VERSION and (not output.exists())


@enforce_types
def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
    """download reader friendly version using @mozilla/readability"""

    out_dir = Path(out_dir or link.link_dir)
    output_folder = out_dir.absolute() / "readability"
    output = str(output_folder)

    # Readability Docs: https://github.com/mozilla/readability

    status = 'succeeded'
    cmd = [
        CURL_BINARY,
        link.url
    ]
    timer = TimedProgress(timeout, prefix='      ')
    try:<commit_msg>add comment about fake cmd<commit_after>__package__ = 'archivebox.extractors'

from pathlib import Path
from tempfile import NamedTemporaryFile

from typing import Optional
import json

from ..index.schema import Link, ArchiveResult, ArchiveError
from ..system import run, atomic_write
from ..util import (
    enforce_types,
    download_url,
    is_static_file,
    
)
from ..config import (
    TIMEOUT,
    CURL_BINARY,
    SAVE_READABILITY,
    DEPENDENCIES,
    READABILITY_VERSION,
)
from ..logging_util import TimedProgress

@enforce_types
def get_html(link: Link, path: Path) -> str:
    """
    Try to find wget, singlefile and then dom files.
    If none is found, download the url again.
    """
    canonical = link.canonical_outputs()
    abs_path = path.absolute()
    sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]]
    document = None
    for source in sources:
        try:
            with open(abs_path / source, "r") as f:
                document = f.read()
                break
        except (FileNotFoundError, TypeError):
            continue
    if document is None:
        return download_url(link.url)
    else:
        return document

@enforce_types
def should_save_readability(link: Link, out_dir: Optional[str]=None) -> bool:
    out_dir = out_dir or link.link_dir
    if is_static_file(link.url):
        return False

    output = Path(out_dir or link.link_dir) / 'readability'
    return SAVE_READABILITY and READABILITY_VERSION and (not output.exists())


@enforce_types
def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
    """download reader friendly version using @mozilla/readability"""

    out_dir = Path(out_dir or link.link_dir)
    output_folder = out_dir.absolute() / "readability"
    output = str(output_folder)

    # Readability Docs: https://github.com/mozilla/readability

    status = 'succeeded'
    # fake command to show the user so they have something to try debugging if get_html fails
    cmd = [
        CURL_BINARY,
        link.url
    ]
    timer = TimedProgress(timeout, prefix='      ')
    try:
{'fraction_non_alphanumeric': 0.09112538662859862, 'fraction_numerical': 0.0, 'mean_word_length': 3.0974658869395713, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '804007', 'n_tokens_mistral': 1277, 'n_tokens_neox': 1247, 'n_words': 357}