<commit_before><?php

namespace yii\console\controllers;
use yii\console\Controller;
use yii\console\models\Feed;
use yii\console\models\FeedType;

class FeederController extends Controller {
 
    public function actionIndex() {
        
        $feedCategories = $this->getFeedCategories();
        
        $feeds = $this->retrieveFeeds($feedCategories);
        $this->rememberFeeds($feeds);
        
    }
    
    public function retrieveFeeds($feedCategories){
        $feeds = [];
        $datetime = $this->getLastFeedDate();
        foreach($feedCategories as $cat){
            $content = file_get_contents($cat->link);
            $item = simplexml_load_string($content);
            foreach($item->channel->item as $entry){
                // title, description, link, pubDate, source, guid
                $feed = new Feed();
                $feed->title = $entry->title;
                $feed->description = $entry->description;
                $feed->link = $entry->link;
                $feed->date_posted = date('Y-m-d H:i:s', strtotime($entry->pubDate));
                $feed->typeId = $cat->id;
                
                if($feed->date_posted > $datetime){
                    array_push($feeds, $feed);
                }
            }
        }
        return $feeds;
    }
    
    public function rememberFeeds($feeds){
        $conn = $this->createConnection();
        
        $values = [];
        foreach($feeds as $feed){
            $row = [$feed->title, $feed->description, $feed->date_posted, $feed->link, $feed->typeId];
            array_push($values, $row);
        }

        $conn->createCommand()->batchInsert('feed',
            ['title', 'description', 'date_posted', 'link', 'type_id'],
             $values)->execute();
    }
    
    public function getFeedCategories(){
        $feedCategories = [];
        $conn = $this->createConnection();
        $result = $conn->createCommand('SELECT * FROM feed_type')->queryAll();
        foreach($result as $res){
            $cat = new FeedType();
            $cat->id = $res['id'];
            $cat->title = $res['title'];
            $cat->link = $res['link'];
            array_push($feedCategories, $cat);
        }
        return $feedCategories;
    }
    
    public function getLastFeedDate(){
        $datetime = 0;
        $conn = $this->createConnection();
        $result = $conn->createCommand('SELECT date_posted FROM feed ORDER BY date_posted DESC LIMIT 1')->queryAll();
        foreach($result as $res){
            $datetime = $res['date_posted'];
        }
        return $datetime;
    }

    public function createConnection(){
        return new \yii\db\Connection([
            'dsn' => 'mysql:host=localhost;dbname=bravo_projekat',
            'username' => 'root',
            'password' => '<PASSWORD>',
            'charset' => 'utf8',
        ]);
    }
}<commit_msg>Dodano izvlacenje linka slike iz opisa i uklanjanje tagova iz opisa.<commit_after><?php

namespace yii\console\controllers;
use yii\console\Controller;
use yii\console\models\Feed;
use yii\console\models\FeedType;

class FeederController extends Controller {
 
    public function actionIndex() {
        
        $feedCategories = $this->getFeedCategories();
        
        $feeds = $this->retrieveFeeds($feedCategories);
        $this->rememberFeeds($feeds);
        
    }
    
    public function retrieveFeeds($feedCategories){
        $feeds = [];
        $datetime = $this->getLastFeedDate();
        foreach($feedCategories as $cat){
            $content = file_get_contents($cat->link);
            $item = simplexml_load_string($content);
            foreach($item->channel->item as $entry){
                // title, description, link, pubDate, source, guid
                $entryDate = date('Y-m-d H:i:s', strtotime($entry->pubDate));
                if($entryDate > $datetime){
                    $feed = new Feed();
                    $feed->title = $entry->title;
                    $feed->description = $entry->description;
                    $feed->image_link = '';
                    
                    $matches = [];
                    preg_match('/src="(.*?)"/', $feed->description, $matches);
                    try{
                        $feed->image_link = $matches[1];
                        $pos = strpos($feed->description, "</a>");
                        $temp = substr($feed->description, $pos);
                        $feed->description = strip_tags($temp);
                    } catch(\yii\base\ErrorException $e){
                        //echo "nema slike";
                    }
                    
                    $feed->link = $entry->link;
                    $feed->date_posted = $entryDate;
                    $feed->typeId = $cat->id;
                    
                    array_push($feeds, $feed);
                }
            }
        }
        return $feeds;
    }
    
    public function rememberFeeds($feeds){
        $conn = $this->createConnection();
        
        $values = [];
        foreach($feeds as $feed){
            $row = [$feed->title, $feed->description, $feed->date_posted, $feed->link, $feed->image_link, $feed->typeId];
            array_push($values, $row);
        }

        $conn->createCommand()->batchInsert('feed',
            ['title', 'description', 'date_posted', 'link', 'image_link', 'type_id'],
             $values)->execute();
    }
    
    public function getFeedCategories(){
        $feedCategories = [];
        $conn = $this->createConnection();
        $result = $conn->createCommand('SELECT * FROM feed_type')->queryAll();
        foreach($result as $res){
            $cat = new FeedType();
            $cat->id = $res['id'];
            $cat->title = $res['title'];
            $cat->link = $res['link'];
            array_push($feedCategories, $cat);
        }
        return $feedCategories;
    }
    
    public function getLastFeedDate(){
        $datetime = 0;
        $conn = $this->createConnection();
        $result = $conn->createCommand('SELECT date_posted FROM feed ORDER BY date_posted DESC LIMIT 1')->queryAll();
        foreach($result as $res){
            $datetime = $res['date_posted'];
        }
        return $datetime;
    }

    public function createConnection(){
        return new \yii\db\Connection([
            'dsn' => 'mysql:host=localhost;dbname=bravo_projekat',
            'username' => 'root',
            'password' => '<PASSWORD>',
            'charset' => 'utf8',
        ]);
    }
}
{'fraction_non_alphanumeric': 0.14918057895542963, 'fraction_numerical': 0.0010721396844846072, 'mean_word_length': 1.7775414717141642, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3954400', 'n_tokens_mistral': 1840, 'n_tokens_neox': 1753, 'n_words': 341}