<commit_before><?php

namespace Api;

class PostUrlCheck extends Api
{
    /**
     * GET /
     */
    public function get()
    {
        $url = $this->input->query['url'];

        if (!$url) {
            $this->error('Params "url" required');
        }

        $_payload = $this->scrapePage($url);

        preg_match('/<title>([^<]+)<\/title>/i', $_payload, $title_matches);
        preg_match('/<meta name="description" content="(.*)" \/> /i', $_payload, $description_matches);

        $this->data['title'] = $title_matches[1];
        $this->data['content'] = $description_matches[1];
    }

    /**
     * Scrape page with read limit size socket
     *
     * @param $url
     * @return string
     */
    private function scrapePage($url)
    {
        $urlParts = parse_url(trim($url));
        if (!$urlParts['path']) $urlParts['path'] = '/';

        $fp = fsockopen($urlParts['host'], $urlParts['scheme'] === 'https' ? '443' : '80', $errNo, $errStr, 30);
        $_payload = '';

        if (!$fp) {
            $this->error('Scrape error');
        } else {
            $out = "GET {$urlParts['path']} HTTP/1.1\r\n";
            $out .= "Host: {$urlParts['host']}\r\n";
            $out .= "Connection: Close\r\n\r\n";
            var_dump($out);
            fwrite($fp, $out);
            while (!feof($fp) && strlen($_payload) < 2048) {
                $_payload .= fgets($fp, 1024);
            }
            fclose($fp);
        }

        preg_match('/Location: (.+)/i', $_payload, $redirect_matches);
        if ($redirect_matches[1]) {
            return $this->scrapePage($redirect_matches[1]);
        }

        return $_payload;
    }
}<commit_msg>Remove out<commit_after><?php

namespace Api;

class PostUrlCheck extends Api
{
    /**
     * GET /
     */
    public function get()
    {
        $url = $this->input->query['url'];

        if (!$url) {
            $this->error('Params "url" required');
        }

        $_payload = $this->scrapePage($url);

        preg_match('/<title>([^<]+)<\/title>/i', $_payload, $title_matches);
        preg_match('/<meta name="description" content="(.*)" \/> /i', $_payload, $description_matches);

        $this->data['title'] = $title_matches[1];
        $this->data['content'] = $description_matches[1];
    }

    /**
     * Scrape page with read limit size socket
     *
     * @param $url
     * @return string
     */
    private function scrapePage($url)
    {
        $urlParts = parse_url(trim($url));
        if (!$urlParts['path']) $urlParts['path'] = '/';

        $fp = fsockopen($urlParts['host'], $urlParts['scheme'] === 'https' ? '443' : '80', $errNo, $errStr, 30);
        $_payload = '';

        if (!$fp) {
            $this->error('Scrape error');
        } else {
            $out = "GET {$urlParts['path']} HTTP/1.1\r\n";
            $out .= "Host: {$urlParts['host']}\r\n";
            $out .= "Connection: Close\r\n\r\n";
            fwrite($fp, $out);
            while (!feof($fp) && strlen($_payload) < 2048) {
                $_payload .= fgets($fp, 1024);
            }
            fclose($fp);
        }

        preg_match('/Location: (.+)/i', $_payload, $redirect_matches);
        if ($redirect_matches[1]) {
            return $this->scrapePage($redirect_matches[1]);
        }

        return $_payload;
    }
}
{'fraction_non_alphanumeric': 0.2023486901535682, 'fraction_numerical': 0.012646793134598013, 'mean_word_length': 1.9954914337240757, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2857030', 'n_tokens_mistral': 1098, 'n_tokens_neox': 1054, 'n_words': 196}