<commit_before><?php
/**
 * ownCloud - News
 *
 * This file is licensed under the Affero General Public License version 3 or
 * later. See the COPYING file.
 *
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 * @copyright <NAME> 2012
 * @copyright <NAME> 2012, 2014
 */

namespace OCA\News\ArticleEnhancer;

use \DOMDocument;
use \DOMXpath;

use \ZendXml\Security;
use \OCA\News\Utility\PicoFeedClientFactory;

use \OCA\News\Db\Item;

class XPathArticleEnhancer implements ArticleEnhancer {

    private $maximumTimeout;
    private $clientFactory;
    private $regexXPathPair;


    /**
     * @param \Utility\PicoFeedClientFactory $clientFactory
     * @param array $regexXPathPair an associative array containing regex to
     * match the url and the xpath that should be used for it to extract the
     * page
     */
    public function __construct(PicoFeedClientFactory $clientFactory,
                                array $regexXPathPair){
        $this->clientFactory = $clientFactory;
        $this->regexXPathPair = $regexXPathPair;
    }

    /**
     * @param \OCA\News\Db\Item $item
     * @return \OCA\News\Db\Item enhanced item
     */
    public function enhance(Item $item){

        foreach($this->regexXPathPair as $regex => $search) {

            if(preg_match($regex, $item->getUrl())) {
                list($body, $contentType) = $this->getFile($item->getUrl());
                if(preg_match('/(?<=charset=)[^;]*/', $body, $matches)) {
                    $encoding = $matches[0];
                    $body = mb_convert_encoding($body, 'HTML-ENTITIES',
                                                strtoupper($encoding));
                }

                $dom = new DOMDocument();

                Security::scan($body, $dom, function ($xml, $dom) {
                    return @$dom->loadHTML($xml, LIBXML_NONET);
                });
<commit_msg>try to fix encoding<commit_after><?php
/**
 * ownCloud - News
 *
 * This file is licensed under the Affero General Public License version 3 or
 * later. See the COPYING file.
 *
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 * @copyright <NAME> 2012
 * @copyright <NAME> 2012, 2014
 */

namespace OCA\News\ArticleEnhancer;

use \DOMDocument;
use \DOMXpath;

use \ZendXml\Security;
use \PicoFeed\Encoding\Encoding;

use \OCA\News\Utility\PicoFeedClientFactory;

use \OCA\News\Db\Item;

class XPathArticleEnhancer implements ArticleEnhancer {

    private $maximumTimeout;
    private $clientFactory;
    private $regexXPathPair;


    /**
     * @param \Utility\PicoFeedClientFactory $clientFactory
     * @param array $regexXPathPair an associative array containing regex to
     * match the url and the xpath that should be used for it to extract the
     * page
     */
    public function __construct(PicoFeedClientFactory $clientFactory,
                                array $regexXPathPair){
        $this->clientFactory = $clientFactory;
        $this->regexXPathPair = $regexXPathPair;
    }

    /**
     * @param \OCA\News\Db\Item $item
     * @return \OCA\News\Db\Item enhanced item
     */
    public function enhance(Item $item){

        foreach($this->regexXPathPair as $regex => $search) {

            if(preg_match($regex, $item->getUrl())) {
                list($body, $httpEncoding) = $this->getFile($item->getUrl());
                if(preg_match('/(?<=charset=)[^;]*/', $body, $matches)) {
                    $encoding = $matches[0];
                    $body = Encoding::convert($body, $encoding);
                    $body = mb_convert_encoding($body, 'HTML-ENTITIES',
                                                strtoupper($encoding));
                }

                $dom = new DOMDocument();

                Security::scan($body, $dom, function ($xml, $dom) {
                    return @$dom->loadHTML($xml, LIBXML_NONET);
                });

{'fraction_non_alphanumeric': 0.12745098039215685, 'fraction_numerical': 0.007223942208462332, 'mean_word_length': 2.241638795986622, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3211431', 'n_tokens_mistral': 1178, 'n_tokens_neox': 1108, 'n_words': 276}