<commit_before><?php

namespace Error;

/**
 * Description of Checker Main
 *
 * @author bootjp
 */
class Checker
{
    protected $client;

    protected $contentsSize;

    protected $doubleCheck;

    protected $garbage = [];

    /**
     * initialisation.
     * @param int  $contentSize [optional]
     * @param bool $doubleCheck [optional]
     */
    public function __construct($auth = null, $contentSize = 500, $doubleCheck = true)
    {
        $this->contentsSize = (int) $contentSize;
        $this->doubleCheck = (bool) $doubleCheck;
        $this->client = new \GuzzleHttp\Client(
            ['defaults' =>
                ['exceptions' => false],
                ['headers' => [
                    'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) '
                  . 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36']
                ]
            ]
        );
        if (!is_null($auth)) {
            list($username, $password) = explode(':', $auth, 2);
            $this->client->setDefaultOption('auth', [$username, $password]);
        }
    }
    /**
     * Wrapper
     * @param  mixed $url      [require]
     * @param  bool $getFlag   [optional] true when fetch content on the $url
     * @param  bool $recursion [optional] true when fetch content on the link recursion.
     * @throws ReflectionException
     * @return array URLLIST
     */
    public function start($url, $flag = 'true:false')
    {
        $urlList = [];
        $result['white'] = [];
        $result['black'] = [];
        list((bool) $getFlag, (bool) $recursion) = explode(':', $flag, 2);

        if ($getFlag) {
            $url = $this->fetchByContents($url);

            if ($recursion) {
                $urlList = array_map(function($uri) {
                    return $this->fetchByContents($uri);
                }, $url);

                $url = $this->urlFilter($urlList);
            }
        }

        if (is_null($url)) {
            throw new \ReflectionException('Start URL is not null.');
        } else if (is_array($url)) {
            $urlList = $this->urlFilter($url);
        } else if (is_string($value)) {
            $urlList[] = $url;
        } else if (is_object($value)) {
            $urlList[] = (string) $url;
        }

        echo 'Cheking..';

        foreach ($urlList as $key => $url) {
            $metaData = $this->client->get($url);
            $hardCheck = (array) $this->hardCheckByHeader($metaData);
            $softCheck = (array) $this->softCheckByContents($metaData);

            if ($hardCheck['result'] && $softCheck['result']) {
                $result['white'][$key]['url'] = $url;
                $result['white'][$key]['status'] = 'OK';
            } else {
                $result['black'][$key]['url'] = $url;
                $result['black'][$key]['status'] = array_key_exists('status', $hardCheck) ? $hardCheck['status'] : $softCheck['status'];
            }

            sleep(1);
            echo '.';
        }
        $result['UnknownLinks'] = $this->garbage;

        return $result;
    }

    /**
     * Fetch Page Contents Links
     * @param  mixed $baseUrl
     * @return array URllist
     */
    private function fetchByContents($baseUrl)
    {
        $urlList = [];
        $matches = [];
        $urlList['baseUrl'] = (string) $baseUrl;

        $contents = $this->client->get($baseUrl)->getBody()->getContents();

        preg_match_all('{<a.+?href=[\"|\'](?<url>.+?)[\"\|\'].*?>}is', $contents, $matches);

        if (!array_key_exists('url', $matches)) {
            throw new \ErrorException('Not match contents on url.');
        }

        foreach ($matches['url'] as $url) {

            if (preg_match('{https?://[\w/:%#\$&\?\(\)~\.=\+\-]+}i', $url)) {
                $urlList[] = $url;
            } else if (preg_match('{https?:\/\/[\w/:%#\$&\?\(\)~\.=\+\-]+}i', $baseUrl . $url)) {
                if (preg_match("{(^#[A-Z0-9].+?$)}i", $url)) {
                    $this->garbage[] = $url;
                } else {
                    $urlList[] = $baseUrl . $url;
                }
            } else {
                $this->garbage[] = $url;
            }
        }

        return array_unique($urlList);
    }

    /**<commit_msg>bug fix<commit_after><?php

namespace Error;

/**
 * Description of Checker Main
 *
 * @author bootjp
 */
class Checker
{
    protected $client;

    protected $contentsSize;

    protected $doubleCheck;

    protected $garbage = [];

    /**
     * initialisation.
     * @param int  $contentSize [optional]
     * @param bool $doubleCheck [optional]
     */
    public function __construct($auth = null, $contentSize = 500, $doubleCheck = true)
    {
        $this->contentsSize = (int) $contentSize;
        $this->doubleCheck = (bool) $doubleCheck;
        $this->client = new \GuzzleHttp\Client(
            ['defaults' =>
                ['exceptions' => false],
                ['headers' => [
                    'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) '
                  . 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36']
                ]
            ]
        );
        if (!is_null($auth)) {
            list($username, $password) = explode(':', $auth, 2);
            $this->client->setDefaultOption('auth', [$username, $password]);
        }
    }
    /**
     * Wrapper
     * @param  mixed $url      [require]
     * @param  bool $getFlag   [optional] true when fetch content on the $url
     * @param  bool $recursion [optional] true when fetch content on the link recursion.
     * @throws ReflectionException
     * @return array URLLIST
     */
    public function start($url, $flag = 'true:false')
    {
        $urlList = [];
        $result['white'] = [];
        $result['black'] = [];
        list($getFlag, $recursion) = explode(':', $flag, 2);

        if ((bool) $getFlag) {
            echo 'Contents fetching..';
            $url = $this->fetchByContents($url);

            if ((bool) $recursion) {
                $urlList = array_map(function($uri) {
                    return $this->fetchByContents($uri);
                }, $url);

                $url = $this->urlFilter($urlList);
            }
        }

        if (is_null($url)) {
            throw new \ReflectionException('Start URL is not null.');
        } else if (is_array($url)) {
            $urlList = $this->urlFilter($url);
        } else if (is_string($value)) {
            $urlList[] = $url;
        } else if (is_object($value)) {
            $urlList[] = (string) $url;
        }

        echo "\n";
        echo 'Cheking..';

        foreach ($urlList as $key => $url) {
            $metaData = $this->client->get($url);
            $hardCheck = (array) $this->hardCheckByHeader($metaData);
            $softCheck = (array) $this->softCheckByContents($metaData);

            if ($hardCheck['result'] && $softCheck['result']) {
                $result['white'][$key]['url'] = $url;
                $result['white'][$key]['status'] = 'OK';
            } else {
                $result['black'][$key]['url'] = $url;
                $result['black'][$key]['status'] = array_key_exists('status', $hardCheck) ? $hardCheck['status'] : $softCheck['status'];
            }

            usleep(500000);
            echo '.';
        }
        $result['UnknownLinks'] = $this->garbage;

        return $result;
    }

    /**
     * Fetch Page Contents Links
     * @param  mixed $baseUrl
     * @return array URllist
     */
    private function fetchByContents($baseUrl)
    {
        $urlList = [];
        $matches = [];
        $urlList['baseUrl'] = (string) $baseUrl;

        $contents = $this->client->get($baseUrl)->getBody()->getContents();

        preg_match_all('{<a.+?href=[\"|\'](?<url>.+?)[\"\|\'].*?>}is', $contents, $matches);

        if (!array_key_exists('url', $matches)) {
            throw new \ErrorException('Not match contents on url.');
        }

        foreach ($matches['url'] as $url) {

            if (preg_match('{https?://[\w/:%#\$&\?\(\)~\.=\+\-]+}i', $url)) {
                $urlList[] = $url;
            } else if (preg_match('{https?:\/\/[\w/:%#\$&\?\(\)~\.=\+\-]+}i', $baseUrl . $url)) {
                if (preg_match("{(^#[A-Z0-9].+?$)}i", $url)) {
                    $this->garbage[] = $url;
                } else {
                    $urlList[] = $baseUrl . $url;
                }
            } else {
                $this->garbage[] = $url;
            }

            usleep(500000);
            echo '.';
        }

        return array_unique($urlList);
    }

    /**
{'fraction_non_alphanumeric': 0.18482535276428405, 'fraction_numerical': 0.009368494101318528, 'mean_word_length': 1.7599744653686562, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5119985', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2596, 'n_words': 520}