<commit_before>__package__ = 'archivebox.extractors'

import os
from pathlib import Path

from typing import Optional

from ..index.schema import Link, ArchiveResult, ArchiveError
from ..system import run, chmod_file
from ..util import (
    enforce_types,
)
from ..config import (
    TIMEOUT,
    SAVE_SINGLEFILE,
    SINGLEFILE_BINARY,
    SINGLEFILE_VERSION,
    CHROME_BINARY,
)
from ..logging_util import TimedProgress


@enforce_types
def should_save_singlefile(link: Link, out_dir: Optional[str]=None) -> bool:
    out_dir = out_dir or link.link_dir
    if not os.path.exists(out_dir):
        return False

    return SAVE_SINGLEFILE


@enforce_types
def save_singlefile(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
    """download full site using single-file"""

    out_dir = out_dir or link.link_dir
    output = str(Path(out_dir).absolute() / "single-file.html")

    # WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
    cmd = [
        SINGLEFILE_BINARY,
        '--browser-executable-path={}'.format(CHROME_BINARY),
        link.url,
        output
    ]

    status = 'succeeded'
    timer = TimedProgress(timeout, prefix='      ')
    try:
        result = run(cmd, cwd=out_dir, timeout=timeout)
<commit_msg>fix: Commit code review suggestions<commit_after>__package__ = 'archivebox.extractors'

import os
from pathlib import Path

from typing import Optional

from ..index.schema import Link, ArchiveResult, ArchiveError
from ..system import run, chmod_file
from ..util import (
    enforce_types,
)
from ..config import (
    TIMEOUT,
    SAVE_SINGLEFILE,
    SINGLEFILE_BINARY,
    SINGLEFILE_VERSION,
    CHROME_BINARY,
)
from ..logging_util import TimedProgress


@enforce_types
def should_save_singlefile(link: Link, out_dir: Optional[str]=None) -> bool:
    out_dir = out_dir or link.link_dir

    output = Path(out_dir or link.link_dir) / 'single-file.html'
    return SAVE_SINGLEFILE and (not output.exists())


@enforce_types
def save_singlefile(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
    """download full site using single-file"""

    out_dir = out_dir or link.link_dir
    output = str(Path(out_dir).absolute() / "single-file.html")

    # SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli
    cmd = [
        SINGLEFILE_BINARY,
        '--browser-executable-path={}'.format(CHROME_BINARY),
        link.url,
        output
    ]

    status = 'succeeded'
    timer = TimedProgress(timeout, prefix='      ')
    try:
        result = run(cmd, cwd=out_dir, timeout=timeout)

{'fraction_non_alphanumeric': 0.10714285714285714, 'fraction_numerical': 0.0, 'mean_word_length': 3.500854700854701, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1767502', 'n_tokens_mistral': 882, 'n_tokens_neox': 858, 'n_words': 201}