<commit_before><?php
require_once(__DIR__ . '/BridgeInterface.php');
/**
 * Extension of BridgeAbstract allowing caching of files downloaded over http.
 * TODO allow file cache invalidation by touching files on access, and removing
 * files/directories which have not been touched since ... a long time
 */
abstract class HttpCachingBridgeAbstract extends BridgeAbstract {

    /**
     * Maintain locally cached versions of pages to download, to avoid multiple downloads.
     * @param url url to cache
     * @param duration duration of the cache file in seconds (default: 24h/86400s)
     * @return content of the file as string
     */
    public function get_cached($url, $duration = 86400){
        $filepath = $this->buildCacheFilePath($url);

        if(file_exists($filepath) && filectime($filepath) < time() - $duration){
            $this->debugMessage('Cache file ' . $filepath . ' exceeded duration of ' . $duration . ' seconds.');
            unlink ($filepath);
            $this->debugMessage('Cached file deleted: ' . $filepath);
        }

        if(file_exists($filepath)){
            $this->debugMessage('loading cached file from ' . $filepath . ' for page at url ' . $url);
            // TODO touch file and its parent, and try to do neighbour deletion
            touch($filepath);
            $content = file_get_contents($filepath);
        } else {
            $this->debugMessage('we have no local copy of ' . $url . ' Downloading to ' . $filepath);
            $dir = substr($filepath, 0, strrpos($filepath, '/'));

            if(!is_dir($dir)){
                $this->debugMessage('creating directories for ' . $dir);
                mkdir($dir, 0777, true);
            }

            $content = $this->getContents($url);
            if($content !== false){
                file_put_contents($filepath, $content);
            }
        }

        return str_get_html($content);
    }

     public function get_cached_time($url){
        $filepath = $this->buildCacheFilePath($url);

        if(!file_exists($filepath)){
            $this->get_cached($url);
        }

        return filectime($filepath);
    }

    private function buildCacheFilePath($url){
        return __DIR__ . '/../cache/pages/' . sha1($url) . '.cache';
    }

    public function remove_from_cache($url){
        $filepath = $this->buildCacheFilePath($url);
        $this->debugMessage('removing from cache \'' . $filepath . '\'');
        unlink($filepath);
    }
}<commit_msg>[HttpCachingBridgeAbstract] Remove get_cached_time and remove_from_cache<commit_after><?php
require_once(__DIR__ . '/BridgeInterface.php');
/**
 * Extension of BridgeAbstract allowing caching of files downloaded over http.
 * TODO allow file cache invalidation by touching files on access, and removing
 * files/directories which have not been touched since ... a long time
 */
abstract class HttpCachingBridgeAbstract extends BridgeAbstract {

    /**
     * Maintain locally cached versions of pages to download, to avoid multiple downloads.
     * @param url url to cache
     * @param duration duration of the cache file in seconds (default: 24h/86400s)
     * @return content of the file as string
     */
    public function get_cached($url, $duration = 86400){
        $filepath = $this->buildCacheFilePath($url);

        if(file_exists($filepath) && filectime($filepath) < time() - $duration){
            $this->debugMessage('Cache file ' . $filepath . ' exceeded duration of ' . $duration . ' seconds.');
            unlink ($filepath);
            $this->debugMessage('Cached file deleted: ' . $filepath);
        }

        if(file_exists($filepath)){
            $this->debugMessage('loading cached file from ' . $filepath . ' for page at url ' . $url);
            // TODO touch file and its parent, and try to do neighbour deletion
            touch($filepath);
            $content = file_get_contents($filepath);
        } else {
            $this->debugMessage('we have no local copy of ' . $url . ' Downloading to ' . $filepath);
            $dir = substr($filepath, 0, strrpos($filepath, '/'));

            if(!is_dir($dir)){
                $this->debugMessage('creating directories for ' . $dir);
                mkdir($dir, 0777, true);
            }

            $content = $this->getContents($url);
            if($content !== false){
                file_put_contents($filepath, $content);
            }
        }

        return str_get_html($content);
    }

    private function buildCacheFilePath($url){
        return __DIR__ . '/../cache/pages/' . sha1($url) . '.cache';
    }
}
{'fraction_non_alphanumeric': 0.12657677250978686, 'fraction_numerical': 0.0078294910830796, 'mean_word_length': 2.4117210682492582, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7466030', 'n_tokens_mistral': 1279, 'n_tokens_neox': 1213, 'n_words': 353}