<commit_before><?php

namespace Zrashwani\NewsScrapper\Adapters;

use \Symfony\Component\DomCrawler\Crawler;

/**
 * Base class that defines skeleton of the any adapter implemented
 *
 * @author <NAME> <<EMAIL>>
 */
abstract class AbstractAdapter
{
    public $currentUrl;

    abstract public function extractTitle(Crawler $crawler);

    abstract public function extractImage(Crawler $crawler);

    abstract public function extractDescription(Crawler $crawler);

    abstract public function extractKeywords(Crawler $crawler);

    abstract public function extractBody(Crawler $crawler);

    abstract public function extractPublishDate(Crawler $crawler);

    abstract public function extractAuthor(Crawler $crawler);

    /**
     * normalize link and turn it into absolute format
     * @param string $link
     * @param boolean $remove_hashes if set true, hashes will be removed from url
     * @return string
     */
    public function normalizeLink($link, $remove_hashes = false)
    {
        $baseUrl = $this->currentUrl;
        if (preg_match('@^http(s?)://.*$@', $baseUrl) === 0 && //local environment assumed here
            preg_match('@^http(s?)://.*$@', $link) === 0) {
                $link = pathinfo($baseUrl, PATHINFO_DIRNAME).'/'.$link;
        } elseif (preg_match('@^http(s?)://.*$@', $link) === 0) { //is not absolute
            $urlParts = parse_url($baseUrl);
            $scheme = isset($urlParts['scheme'])===true?$urlParts['scheme']:'http';
            $host = isset($urlParts['host'])===true?$urlParts['host']:'';
            if (strpos($link, '//') === 0) { //begins with //
                $link = $scheme . ':' . $link;
            } elseif (strpos($link, '/') === 0) { //begins with /
                $link = $scheme.'://'.$host.$link;
            } else {
                $path = isset($urlParts['path'])===true?$urlParts['path']:'/';
                $link = $scheme.'://'.$host.'/'.$link;
            }
        }
        
        if ($remove_hashes === true) {
            $link = preg_replace('@#.*$@', '', $link);
        }
        return $link;
    }

    /**
     * normalizing html scrapped by removing unwanted tags (ex. script, css)
     * and amending external resources paths
     * @param string $raw_html
     * @return string
     */
    public function normalizeHtml($raw_html)
    {<commit_msg>remove unused variable<commit_after><?php

namespace Zrashwani\NewsScrapper\Adapters;

use \Symfony\Component\DomCrawler\Crawler;

/**
 * Base class that defines skeleton of the any adapter implemented
 *
 * @author <NAME> <<EMAIL>>
 */
abstract class AbstractAdapter
{
    public $currentUrl;

    abstract public function extractTitle(Crawler $crawler);

    abstract public function extractImage(Crawler $crawler);

    abstract public function extractDescription(Crawler $crawler);

    abstract public function extractKeywords(Crawler $crawler);

    abstract public function extractBody(Crawler $crawler);

    abstract public function extractPublishDate(Crawler $crawler);

    abstract public function extractAuthor(Crawler $crawler);

    /**
     * normalize link and turn it into absolute format
     * @param string $link
     * @param boolean $remove_hashes if set true, hashes will be removed from url
     * @return string
     */
    public function normalizeLink($link, $remove_hashes = false)
    {
        $baseUrl = $this->currentUrl;
        if (preg_match('@^http(s?)://.*$@', $baseUrl) === 0 && //local environment assumed here
            preg_match('@^http(s?)://.*$@', $link) === 0) {
                $link = pathinfo($baseUrl, PATHINFO_DIRNAME).'/'.$link;
        } elseif (preg_match('@^http(s?)://.*$@', $link) === 0) { //is not absolute
            $urlParts = parse_url($baseUrl);
            $scheme = isset($urlParts['scheme'])===true?$urlParts['scheme']:'http';
            $host = isset($urlParts['host'])===true?$urlParts['host']:'';
            if (strpos($link, '//') === 0) { //begins with //
                $link = $scheme . ':' . $link;
            } elseif (strpos($link, '/') === 0) { //begins with /
                $link = $scheme.'://'.$host.$link;
            } else {                
                $link = $scheme.'://'.$host.'/'.$link;
            }
        }
        
        if ($remove_hashes === true) {
            $link = preg_replace('@#.*$@', '', $link);
        }
        return $link;
    }

    /**
     * normalizing html scrapped by removing unwanted tags (ex. script, css)
     * and amending external resources paths
     * @param string $raw_html
     * @return string
     */
    public function normalizeHtml($raw_html)
    {
{'fraction_non_alphanumeric': 0.1582687338501292, 'fraction_numerical': 0.002153316106804479, 'mean_word_length': 2.5162755488266466, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '361650', 'n_tokens_mistral': 1415, 'n_tokens_neox': 1376, 'n_words': 336}