<commit_before><?php
class Vps_Util_Model_Feed_Row_Feed extends Vps_Model_Row_Data_Abstract
{
    private $_xml;
    private $_entries;
    const FORMAT_RSS = 'rss';
    const FORMAT_ATOM = 'atom';
    public function __construct($config)
    {
        $data['url'] = $config['url'];
        $encoding = false;

        if (substr($data['url'], 0, 7)=='file://') {
            $str = file_get_contents($data['url']);
        } else {
            $client = new Zend_Http_Client($data['url'], array(
                //TODO: aus config auslesen!
                'useragent' => 'User-Agent: RSSIncludeBot/1.0 (http://www.rssinclude.com/spider)'
            ));
            $response = $client->request();
            if ($response->getStatus() != 200) {
                throw new Vps_Exception("invalid status response");
            }
            $str = $response->getBody();
        }
        $str = trim($str);
        if (preg_match('#<?xml[^>]* encoding=["\']([^"\']*)["\']#', $str, $m)) {
            $encoding = trim(strtolower($m[1]));
            if ($encoding != 'utf8' && $encoding != 'utf-8') {
                $str = iconv($encoding, 'utf-8', $str);
                $str = preg_replace('#(<?xml[^>]* encoding=["\'])([^"\']*)(["\'])#', '\1utf-8\3', $str);
            }
        } else {
            foreach ($http_response_header as $h) {
                if (substr(strtolower($h), 0, 14) == 'content-type: ') {
                    if (preg_match('#charset=([^;]*)#', strtolower($h), $m)) {
                        $encoding = trim($m[1]);
                        if ($encoding != 'utf8' && $encoding != 'utf-8') {
                            $str = iconv($encoding, 'utf-8', $str);
                        }
                    }
                    break;
                }
            }
        }
        if (!$encoding) {
            $encoding = 'iso-8859-1';
            $str = iconv($encoding, 'utf-8', $str);
        }
        Vps_Benchmark::count('loaded feed');
        $this->_xml = simplexml_load_string($str, 'SimpleXMLElement', LIBXML_NOERROR|LIBXML_NOWARNING);
        if (!$this->_xml) {
            //try with another encoding
            $this->_xml = simplexml_load_string(iconv('iso-8859-1', 'utf-8', $str), 'SimpleXMLElement', LIBXML_NOERROR|LIBXML_NOWARNING);
            if ($this->_xml) {
                $encoding = 'iso-8859-1';
            }
        }
<commit_msg>bugfix, wg. umstellung auf zend_http_client gibts  nicht mehr<commit_after><?php
class Vps_Util_Model_Feed_Row_Feed extends Vps_Model_Row_Data_Abstract
{
    private $_xml;
    private $_entries;
    const FORMAT_RSS = 'rss';
    const FORMAT_ATOM = 'atom';
    public function __construct($config)
    {
        $data['url'] = $config['url'];
        $encoding = false;

        if (substr($data['url'], 0, 7)=='file://') {
            $str = file_get_contents($data['url']);
        } else {
            $client = new Zend_Http_Client($data['url'], array(
                //TODO: aus config auslesen!
                'useragent' => 'User-Agent: RSSIncludeBot/1.0 (http://www.rssinclude.com/spider)'
            ));
            $response = $client->request();

            if ($response->getStatus() != 200) {
                throw new Vps_Exception("invalid status response");
            }
            $str = $response->getBody();
        }
        $str = trim($str);
        if (preg_match('#<?xml[^>]* encoding=["\']([^"\']*)["\']#', $str, $m)) {
            $encoding = trim(strtolower($m[1]));
            if ($encoding != 'utf8' && $encoding != 'utf-8') {
                $str = iconv($encoding, 'utf-8', $str);
                $str = preg_replace('#(<?xml[^>]* encoding=["\'])([^"\']*)(["\'])#', '\1utf-8\3', $str);
            }
        } else if (isset($response)) {
            $ct = $response->getHeader('Content-Type');
            if ($ct) {
                if (preg_match('#charset=([^;]*)#', strtolower($ct), $m)) {
                    $encoding = trim($m[1]);
                    if ($encoding != 'utf8' && $encoding != 'utf-8') {
                        $str = iconv($encoding, 'utf-8', $str);
                    }
                }
            }
        }

        if (!$encoding) {
            $encoding = 'iso-8859-1';
            $str = iconv($encoding, 'utf-8', $str);
        }

        Vps_Benchmark::count('loaded feed');
        $this->_xml = simplexml_load_string($str, 'SimpleXMLElement', LIBXML_NOERROR|LIBXML_NOWARNING);
        if (!$this->_xml) {
            //try with another encoding
            $this->_xml = simplexml_load_string(iconv('iso-8859-1', 'utf-8', $str), 'SimpleXMLElement', LIBXML_NOERROR|LIBXML_NOWARNING);
            if ($this->_xml) {
                $encoding = 'iso-8859-1';
            }
        }

{'fraction_non_alphanumeric': 0.18906414300736069, 'fraction_numerical': 0.015352260778128285, 'mean_word_length': 1.786174575278266, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1663355', 'n_tokens_mistral': 1533, 'n_tokens_neox': 1462, 'n_words': 267}