<commit_before><?php

class FileUtil {

    public static function tempFilenameFromUrl($url) {
        $hostname = parse_url($url, PHP_URL_HOST);
        $hostname = str_replace(".", "_", $hostname);
        $basename = "onetsp_{$hostname}_" . substr(md5($url), 0, 8);
        $filename = sys_get_temp_dir() . "/" . $basename;
        return $filename;
    }
    
    public static function downloadPage($url) {
        $user_agent = "Onetsp-RecipeParser/0.1 (+https://github.com/onetsp/RecipeParser)";

        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_HEADER, 0);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 2);
        curl_setopt($ch, CURLOPT_MAXREDIRS, 4);
        curl_setopt($ch, CURLOPT_USERAGENT, $user_agent);

        $html = curl_exec($ch);
        curl_close($ch);

        return $html;
    }

    public static function downloadRecipeWithCache($url) {
        $cache_ttl = 86400 * 3;

        // Target filename
        $filename = FileUtil::tempFilenameFromUrl($url);

        // Only fetch 1x per day
        if (file_exists($filename)
            && filesize($filename) > 0
            && (time() - filemtime($filename) < $cache_ttl)
        ) {
            Log::notice("Found file in cache: $filename", __CLASS__);
            $html = file_get_contents($filename);

        } else {
            // Fetch and cleanup the HTML
            Log::notice("Downloading recipe from url: $url", __CLASS__);

            $html = FileUtil::downloadPage($url);
            $html = RecipeParser_Text::forceUTF8($html);
            $html = RecipeParser_Text::cleanupClippedRecipeHtml($html);

            // Append some notes to the HTML
            $comments = RecipeParser_Text::getRecipeMetadataComment($url, "curl");
            $html = $comments . "\n\n" . $html;

            Log::notice("Saving recipe to file $filename", __CLASS__);
            file_put_contents($filename, $html);
        }

        return $html;
    }

}<commit_msg>Improved logging of file downloads.<commit_after><?php

class FileUtil {

    public static function tempFilenameFromUrl($url) {
        $hostname = parse_url($url, PHP_URL_HOST);
        $hostname = str_replace(".", "_", $hostname);
        $basename = "onetsp_{$hostname}_" . substr(md5($url), 0, 8);
        $filename = sys_get_temp_dir() . "/" . $basename;
        return $filename;
    }
    
    public static function downloadPage($url) {
        $user_agent = "Onetsp-RecipeParser/0.1 (+https://github.com/onetsp/RecipeParser)";

        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_HEADER, 0);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 2);
        curl_setopt($ch, CURLOPT_MAXREDIRS, 4);
        curl_setopt($ch, CURLOPT_USERAGENT, $user_agent);

        $html = curl_exec($ch);
        curl_close($ch);

        return $html;
    }

    public static function downloadRecipeWithCache($url) {
        $cache_ttl = 86400 * 3;

        // Target filename
        $filename = FileUtil::tempFilenameFromUrl($url);

        // Only fetch 1x per day
        if (file_exists($filename)
            && filesize($filename) > 0
            && (time() - filemtime($filename) < $cache_ttl)
        ) {
            $filesize = filesize($filename);
            Log::notice("Found file in cache: $filename, size is $filesize", __CLASS__);
            $html = file_get_contents($filename);

        } else {
            // Fetch and cleanup the HTML
            Log::notice("Downloading file from url: $url", __CLASS__);

            $html = FileUtil::downloadPage($url);
            $html = RecipeParser_Text::forceUTF8($html);
            $html = RecipeParser_Text::cleanupClippedRecipeHtml($html);

            // Append some notes to the HTML
            $comments = RecipeParser_Text::getRecipeMetadataComment($url, "curl");
            $html = $comments . "\n\n" . $html;

            file_put_contents($filename, $html);
            $filesize = filesize($filename);
            Log::notice("Saved file to $filename, size is $filesize", __CLASS__);
        }

        return $html;
    }

}
{'fraction_non_alphanumeric': 0.13438644688644688, 'fraction_numerical': 0.0086996336996337, 'mean_word_length': 2.4159499609069583, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1627949', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1325, 'n_words': 273}