<commit_before><?php
/**
 * User: simon
 * Date: 19.08.2016
 * Time: 18:05
 */
namespace ShortPixel\persist;

use ShortPixel\ClientException;
use \ShortPixel\Persister;
use \ShortPixel\ShortPixel;
use \ShortPixel\Client;

/**
 * Class TextPersister - save the optimization information in .shortpixel files in the current folder of the images
 * @package ShortPixel\persist
 */
class TextPersister implements Persister {

    private $fp;
    private $options;
    private STATIC $ALLOWED_STATUSES = array('pending', 'success', 'skip', 'deleted');
    private STATIC $ALLOWED_TYPES = array('I', 'D');

    function __construct($options)
    {
        $this->options = $options;
        $this->fp = array();
    }

    function isOptimized($path)
    {
        if(!file_exists($path)) {
            return false;
        }
        $fp = $this->openMetaFile(dirname($path), 'read');
        if(!$fp) {
            return false;
        }

        while (($line = fgets($fp)) !== FALSE) {
            $data = $this->parse($line);
            if($data->file === \ShortPixel\MB_basename($path) && $data->status == 'success' ) {
                return true;
            }
        }
        fclose($fp);

        return false;
    }

    protected function ignored($exclude) {
        return array_values(array_merge(array('.','..','.','..','.shortpixel','.sp-options','.sp-lock','.sp-progress','ShortPixelBackups'), is_array($exclude) ? $exclude : array()));
    }

    /**
     * @param $path - the file path on the local drive
     * @param bool $recurse - boolean - go into subfolders or not
     * @param bool $fileList - return the list of files with optimization status (only current folder, not subfolders)
     * @param array $exclude - array of folder names that you want to exclude from the optimization
     * @param bool $persistPath - the path where to look for the metadata, if different from the $path
     * @param int $recurseDepth - how many subfolders deep to go. Defaults to PHP_INT_MAX
     * @param bool $retrySkipped - if true, all skipped files will be reset to pending with retries = 0
     * @return object|void (object)array('status', 'total', 'succeeded', 'pending', 'same', 'failed')
     * @throws PersistException
     */
    function info($path, $recurse = true, $fileList = false, $exclude = array(), $persistPath = false, $recurseDepth = PHP_INT_MAX, $retrySkipped = false) {
        if($persistPath === false) {
            $persistPath = $path;
        }
        $toClose = false; $persistFolder = false;
        $info = array('status' => 'error', 'message' => "Unknown error, please contact support.", 'code' => -999);

        try {
            if(is_dir($path)) {
                try {
                    $persistFolder = $persistPath;
                    $toClose = $this->openMetaFileIfNeeded($persistFolder);
                    $fp = $this->getMetaFile($persistPath);
                    $dataArr = $this->readMetaFile($fp);
                } catch(ClientException $e) {
                    if(is_dir($persistPath) && file_exists($persistPath . '/' . ShortPixel::opt("persist_name"))) {
                        throw $e; //rethrow, there's a problem with the meta file.
                    }
                    $dataArr = array(); //there's no problem if the metadata file is missing and cannot be created, for the info call
                }

                $info = (object)array('status' => 'pending', 'total' => 0, 'succeeded' => 0, 'pending' => 0, 'same' => 0, 'failed' => 0, 'totalSize'=> 0, 'totalOptimizedSize'=> 0, 'todo' => null);
                $files = scandir($path);
                $ignore = $this->ignored($exclude);

                foreach($files as $file) {
                    $filePath = $path . '/' . $file;
                    $targetFilePath = $persistPath . '/' . $file;
                    if (in_array($file, $ignore)
                        || (!ShortPixel::isProcessable($file) && !is_dir($filePath))
                        || isset($dataArr[$file]) && $dataArr[$file]->status == 'deleted'
                    ) {
                        continue;
                    }
                    if (is_dir($filePath)) {
                        if(!$recurse || $recurseDepth <= 0) continue;
                        $subInfo = $this->info($filePath, $recurse, $fileList, $exclude, $targetFilePath, $recurseDepth - 1);
                        if($subInfo->status == 'error') {
                            $info = $subInfo;
                            break;
                        }
                        $info->total += $subInfo->total;
                        $info->succeeded += $subInfo->succeeded;
                        $info->pending += $subInfo->pending;
                        $info->same += $subInfo->same;
                        $info->failed += $subInfo->failed;
                        $info->totalSize += $subInfo->totalSize;
                        $info->totalOptimizedSize += $subInfo->totalOptimizedSize;
                    }
                    else {
                        $info->total++;
                        if(!isset($dataArr[$file]) || $dataArr[$file]->status == 'pending') {
                            $info->pending++;
                        }
                        elseif(($dataArr[$file]->status == 'success' && filesize($targetFilePath) != $dataArr[$file]->optimizedSize)
                            || ($dataArr[$file]->status == 'skip' &&  ($dataArr[$file]->retries <= ShortPixel::MAX_RETRIES || $retrySkipped))) {
                            //file changed since last optimized, mark it as pending
                            $dataArr[$file]->status = 'pending';
                            if($dataArr[$file]->status == 'skip' && $retrySkipped) {
                                $dataArr[$file]->retries = 0;
                            }
                            $this->updateMeta($dataArr[$file], $fp);
                            $info->pending++;
                        }
                        elseif($dataArr[$file]->status == 'success') {
                            if($dataArr[$file]->percent > 0) {
                                $info->succeeded++;
                                $info->totalOptimizedSize += $dataArr[$file]->optimizedSize;
                                $info->totalSize += round(100.0 * $dataArr[$file]->optimizedSize / (100.0 - $dataArr[$file]->percent));
                            } else {
                                $info->same++;
                            }
                        }
                        elseif($dataArr[$file]->status == 'skip'){
                            $info->failed++;
                        }
                    }
                    if($fileList) $info->fileList = $dataArr;
                }

                if(isset($info->pending) && $info->pending == 0 && $info->status !== 'error') {
                    $info->status = 'success';
                }
                if($info->status !== 'error') {
                    $info->todo = $this->getTodo($path, 1, $exclude, $persistPath, ShortPixel::CLIENT_MAX_BODY_SIZE, $recurseDepth);
                }
            }
            else {
                if(!file_exists($persistPath)) {
                    throw new ClientException("File not found: $persistPath", -15);
                }
                $persistFolder = dirname($persistPath);
                $meta = $toClose = false;
                try {
                    $toClose = $this->openMetaFileIfNeeded($persistFolder);
                    $meta = $this->findMeta($persistPath);
                } catch(ClientException $e) {
                    if(is_dir($persistFolder) && file_exists($persistFolder . '/' . ShortPixel::opt("persist_name"))) {
                        throw $e;
                    }
                }

                if(!$meta) {
                    $info = (object)array('status' => 'pending');
                } else {
                    $info = (object)array('status' => $meta->getStatus());
                }

            }
        }
        catch(ClientException $e) {
            $info = (object)array('status' => 'error', 'message' => $e->getMessage(), 'code' => $e->getCode());
        }
        catch(\Exception $e) { //that should've been a finally but we need to be PHP5.4 compatible...
            if($toClose) {
                $this->closeMetaFile($persistFolder);
            }
            throw $e;
        }
        return $info;
    }

    function getTodo($path, $count, $exclude = array(), $persistPath = false, $maxTotalFileSizeMb = ShortPixel::CLIENT_MAX_BODY_SIZE, $recurseDepth = PHP_INT_MAX)
    {
        if(!file_exists($path) || !is_dir($path)) {
            return array();
        }
        if(!$persistPath) {$persistPath = $path;}

        $toClose = $this->openMetaFileIfNeeded($persistPath);
        $fp = $this->getMetaFile($persistPath);

        $files = scandir($path);
        $dataArr = $this->readMetaFile($fp);

        $results = array();
        $pendingURLs = array();
        $ignore = $this->ignored($exclude);
        $remain = $count;<commit_msg>Fix .shortpixel files not closed on some cases<commit_after><?php
/**
 * User: simon
 * Date: 19.08.2016
 * Time: 18:05
 */
namespace ShortPixel\persist;

use ShortPixel\ClientException;
use \ShortPixel\Persister;
use \ShortPixel\ShortPixel;
use \ShortPixel\Client;

/**
 * Class TextPersister - save the optimization information in .shortpixel files in the current folder of the images
 * @package ShortPixel\persist
 */
class TextPersister implements Persister {

    private $fp;
    private $options;
    private STATIC $ALLOWED_STATUSES = array('pending', 'success', 'skip', 'deleted');
    private STATIC $ALLOWED_TYPES = array('I', 'D');

    function __construct($options)
    {
        $this->options = $options;
        $this->fp = array();
    }

    function isOptimized($path)
    {
        if(!file_exists($path)) {
            return false;
        }
        $fp = $this->openMetaFile(dirname($path), 'read');
        if(!$fp) {
            return false;
        }

        while (($line = fgets($fp)) !== FALSE) {
            $data = $this->parse($line);
            if($data->file === \ShortPixel\MB_basename($path) && $data->status == 'success' ) {
                return true;
            }
        }
        fclose($fp);

        return false;
    }

    protected function ignored($exclude) {
        return array_values(array_merge(array('.','..','.','..','.shortpixel','.sp-options','.sp-lock','.sp-progress','ShortPixelBackups'), is_array($exclude) ? $exclude : array()));
    }

    /**
     * @param $path - the file path on the local drive
     * @param bool $recurse - boolean - go into subfolders or not
     * @param bool $fileList - return the list of files with optimization status (only current folder, not subfolders)
     * @param array $exclude - array of folder names that you want to exclude from the optimization
     * @param bool $persistPath - the path where to look for the metadata, if different from the $path
     * @param int $recurseDepth - how many subfolders deep to go. Defaults to PHP_INT_MAX
     * @param bool $retrySkipped - if true, all skipped files will be reset to pending with retries = 0
     * @return object|void (object)array('status', 'total', 'succeeded', 'pending', 'same', 'failed')
     * @throws PersistException
     */
    function info($path, $recurse = true, $fileList = false, $exclude = array(), $persistPath = false, $recurseDepth = PHP_INT_MAX, $retrySkipped = false) {
        if($persistPath === false) {
            $persistPath = $path;
        }
        $toClose = false; $persistFolder = false;
        $info = array('status' => 'error', 'message' => "Unknown error, please contact support.", 'code' => -999);

        try {
            if(is_dir($path)) {
                try {
                    $persistFolder = $persistPath;
                    $toClose = $this->openMetaFileIfNeeded($persistFolder);
                    $fp = $this->getMetaFile($persistPath);
                    $dataArr = $this->readMetaFile($fp);
                } catch(ClientException $e) {
                    if(is_dir($persistPath) && file_exists($persistPath . '/' . ShortPixel::opt("persist_name"))) {
                        throw $e; //rethrow, there's a problem with the meta file.
                    }
                    $dataArr = array(); //there's no problem if the metadata file is missing and cannot be created, for the info call
                }

                $info = (object)array('status' => 'pending', 'total' => 0, 'succeeded' => 0, 'pending' => 0, 'same' => 0, 'failed' => 0, 'totalSize'=> 0, 'totalOptimizedSize'=> 0, 'todo' => null);
                $files = scandir($path);
                $ignore = $this->ignored($exclude);

                foreach($files as $file) {
                    $filePath = $path . '/' . $file;
                    $targetFilePath = $persistPath . '/' . $file;
                    if (in_array($file, $ignore)
                        || (!ShortPixel::isProcessable($file) && !is_dir($filePath))
                        || isset($dataArr[$file]) && $dataArr[$file]->status == 'deleted'
                    ) {
                        continue;
                    }
                    if (is_dir($filePath)) {
                        if(!$recurse || $recurseDepth <= 0) continue;
                        $subInfo = $this->info($filePath, $recurse, $fileList, $exclude, $targetFilePath, $recurseDepth - 1);
                        if($subInfo->status == 'error') {
                            $info = $subInfo;
                            break;
                        }
                        $info->total += $subInfo->total;
                        $info->succeeded += $subInfo->succeeded;
                        $info->pending += $subInfo->pending;
                        $info->same += $subInfo->same;
                        $info->failed += $subInfo->failed;
                        $info->totalSize += $subInfo->totalSize;
                        $info->totalOptimizedSize += $subInfo->totalOptimizedSize;
                    }
                    else {
                        $info->total++;
                        if(!isset($dataArr[$file]) || $dataArr[$file]->status == 'pending') {
                            $info->pending++;
                        }
                        elseif(($dataArr[$file]->status == 'success' && filesize($targetFilePath) != $dataArr[$file]->optimizedSize)
                            || ($dataArr[$file]->status == 'skip' &&  ($dataArr[$file]->retries <= ShortPixel::MAX_RETRIES || $retrySkipped))) {
                            //file changed since last optimized, mark it as pending
                            $dataArr[$file]->status = 'pending';
                            if($dataArr[$file]->status == 'skip' && $retrySkipped) {
                                $dataArr[$file]->retries = 0;
                            }
                            $this->updateMeta($dataArr[$file], $fp);
                            $info->pending++;
                        }
                        elseif($dataArr[$file]->status == 'success') {
                            if($dataArr[$file]->percent > 0) {
                                $info->succeeded++;
                                $info->totalOptimizedSize += $dataArr[$file]->optimizedSize;
                                $info->totalSize += round(100.0 * $dataArr[$file]->optimizedSize / (100.0 - $dataArr[$file]->percent));
                            } else {
                                $info->same++;
                            }
                        }
                        elseif($dataArr[$file]->status == 'skip'){
                            $info->failed++;
                        }
                    }
                    if($fileList) $info->fileList = $dataArr;
                }

                if(isset($info->pending) && $info->pending == 0 && $info->status !== 'error') {
                    $info->status = 'success';
                }
                if($info->status !== 'error') {
                    $info->todo = $this->getTodo($path, 1, $exclude, $persistPath, ShortPixel::CLIENT_MAX_BODY_SIZE, $recurseDepth);
                }
            }
            else {
                if(!file_exists($persistPath)) {
                    throw new ClientException("File not found: $persistPath", -15);
                }
                $persistFolder = dirname($persistPath);
                $meta = $toClose = false;
                try {
                    $toClose = $this->openMetaFileIfNeeded($persistFolder);
                    $meta = $this->findMeta($persistPath);
                } catch(ClientException $e) {
                    if(is_dir($persistFolder) && file_exists($persistFolder . '/' . ShortPixel::opt("persist_name"))) {
                        throw $e;
                    }
                }

                if(!$meta) {
                    $info = (object)array('status' => 'pending');
                } else {
                    $info = (object)array('status' => $meta->getStatus());
                }

            }
            if($toClose) {
                $this->closeMetaFile($persistFolder);
            }
        }
        catch(ClientException $e) {
            if($toClose) {
                $this->closeMetaFile($persistFolder);
            }
            $info = (object)array('status' => 'error', 'message' => $e->getMessage(), 'code' => $e->getCode());
        }
        catch(\Exception $e) { //that should've been a finally but we need to be PHP5.4 compatible...
            if($toClose) {
                $this->closeMetaFile($persistFolder);
            }
            throw $e;
        }
        return $info;
    }

    function getTodo($path, $count, $exclude = array(), $persistPath = false, $maxTotalFileSizeMb = ShortPixel::CLIENT_MAX_BODY_SIZE, $recurseDepth = PHP_INT_MAX)
    {
        if(!file_exists($path) || !is_dir($path)) {
            return array();
        }
        if(!$persistPath) {$persistPath = $path;}

        $toClose = $this->openMetaFileIfNeeded($persistPath);
        $fp = $this->getMetaFile($persistPath);

        $files = scandir($path);
        $dataArr = $this->readMetaFile($fp);

        $results = array();
        $pendingURLs = array();
        $ignore = $this->ignored($exclude);
        $remain = $count;
{'fraction_non_alphanumeric': 0.14584348767939345, 'fraction_numerical': 0.004440834010289737, 'mean_word_length': 1.5593901593901593, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 177, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '398952', 'n_tokens_mistral': 5142, 'n_tokens_neox': 4864, 'n_words': 1046}