<commit_before><?php



namespace App\Console\Commands;

use App\Item;
use Illuminate\Support\Facades\App;
use Illuminate\Console\Command;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputArgument;

class OaiPmhDownloadImages extends Command
{

    /**
     * The console command name.
     *
     * @var string
     */
    protected $name = 'oai-pmh:download-images';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Download images for items.';

    protected $log;

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function fire()
    {
        $pocet = Item::where('img_url', '!=', '')->where('has_image', '=', 0)->count();

        if (! $this->confirm("Naozaj spustit stahovanie obrazkov pre {$pocet} diel? [yes|no]", true)) {
            $this->comment('Tak dovidenia.');
            return;
        }

        $logFile = 'image_download.log';
        $this->log = new \Monolog\Logger('image_download');
        $this->log->pushHandler(new \Monolog\Handler\StreamHandler(storage_path().'/logs/'.$logFile, \Monolog\Logger::WARNING));

        $i = 0;
        $failures = 0;

        Item::where('img_url', '!=', '')->where('has_image', '=', 0)->chunkById(200, function ($items) use (&$i, &$failures) {
            // $items->load('authorities');
            foreach ($items as $item) {
                if ($item::hasImageForId($item->id) || $this->downloadImage($item)) {
                    $i++;
                    $item->has_image = true;
                    $item->save();
                } else {
                    $failures++;
                }
                
                if (App::runningInConsole()) {
                    if ($i % 100 == 0) {
                        echo date('h:i:s'). " " . $i . "\n";
                    }
                }
            }
        });
        $message = 'Pre ' . $i . ' diel boli stiahnute obrazky';
        if ($failures > 0) {
            $message .= "\n" . $failures . ' diela nemajú žiadny obrázok alebo sa nepodarilo sťahovať zdroj';
        }
        $this->comment($message);
        return true;
    }

    /**
     * Get the console command arguments.
     *
     * @return array
     */
    protected function getArguments()
    {
        return array(
            // array('example', InputArgument::REQUIRED, 'An example argument.'),
        );
    }

    /**
     * Get the console command options.
     *
     * @return array
     */
    protected function getOptions()
    {
        return array(
            // array('example', null, InputOption::VALUE_OPTIONAL, 'An example option.', null),
        );
    }

    private function downloadImage($item)
    {
        $file = $item->img_url;
        try {
            $data = file_get_contents($file);
        } catch (\Exception $e) {
            $this->log->addError($item->img_url . ': ' . $e->getMessage());
            return false;
        }
        
        $full = true;
        if ($new_file = $item->getImagePath($full)) {
            file_put_contents($new_file, $data);
            return true;
        }
        return false;
    }
}<commit_msg>add comments and make method name more descriptive<commit_after><?php



namespace App\Console\Commands;

use App\Item;
use Illuminate\Support\Facades\App;
use Illuminate\Console\Command;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputArgument;

class OaiPmhDownloadImages extends Command
{

    /**
     * The console command name.
     *
     * @var string
     */
    protected $name = 'oai-pmh:download-images';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Download images for items.';

    protected $log;

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function fire()
    {
        // $pocet = Item::where('img_url', '!=', '')->where('has_image', '=', 0)->count();
        $pocet = Item::whereHas('images', function($q) 
        {
            $q->where('img_url', '!=', '');
        })->where('has_image', '=', 0)->count();

        if (! $this->confirm("Naozaj spustit stahovanie obrazkov pre {$pocet} diel? [yes|no]", true)) {
            $this->comment('Tak dovidenia.');
            return;
        }

        $logFile = 'image_download.log';
        $this->log = new \Monolog\Logger('image_download');
        $this->log->pushHandler(new \Monolog\Handler\StreamHandler(storage_path().'/logs/'.$logFile, \Monolog\Logger::WARNING));

        $i = 0;
        $failures = 0;

        Item::whereHas('images', function($q) 
        {
            $q->where('img_url', '!=', '');
        })->where('has_image', '=', 0)->chunkById(200, function ($items) use (&$i, &$failures) {
            // $items->load('authorities');
            foreach ($items as $item) {
                if ($item::hasImageForId($item->id) || $this->downloadImages($item)) {
                    $i++;
                    $item->has_image = true;
                    $item->save();
                } else {
                    $failures++;
                }
                
                if (App::runningInConsole()) {
                    if ($i % 100 == 0) {
                        echo date('h:i:s'). " " . $i . "\n";
                    }
                }
            }
        });
        $message = 'Pre ' . $i . ' diel boli stiahnute obrazky';
        if ($failures > 0) {
            $message .= "\n" . $failures . ' diela nemajú žiadny obrázok alebo sa nepodarilo sťahovať zdroj';
        }
        $this->comment($message);
        return true;
    }

    /**
     * Get the console command arguments.
     *
     * @return array
     */
    protected function getArguments()
    {
        return array(
            // array('example', InputArgument::REQUIRED, 'An example argument.'),
        );
    }

    /**
     * Get the console command options.
     *
     * @return array
     */
    protected function getOptions()
    {
        return array(
            // array('example', null, InputOption::VALUE_OPTIONAL, 'An example option.', null),
        );
    }

    private function downloadImages($item)
    {
        // downloadImages returns false by default
        $returnBoolean = false;
        foreach ($item->images() as $image) {
            $file = $image->$img_url;
            try {
                $data = file_get_contents($file);
            } catch (\Exception $e) {
                $this->log->addError($image->$img_url . ': ' . $e->getMessage());
                // end the loop and downloadImages still returns false -- all or nothing
                break;
            }
            $full = true;
            if ($new_file = $item->getImagePath($full)) {
                file_put_contents($new_file, $data);
                // if this succeeds we can return true
                $returnBoolean = true;
            }
        };
        return $returnBoolean;    
    }
}
{'fraction_non_alphanumeric': 0.15075172694026817, 'fraction_numerical': 0.0033861573885954218, 'mean_word_length': 1.650394831299354, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6192711', 'n_tokens_mistral': 2205, 'n_tokens_neox': 2128, 'n_words': 459}