<commit_before><?php
namespace Grasshopper\curl;

use Grasshopper\Grasshopper;
use Grasshopper\exception\GrasshopperException;
use Grasshopper\HttpPostRequest;
use Grasshopper\event\SuccessEvent;
use Grasshopper\event\ErrorEvent;

class CurlRequest
{
    /** @var string */
    private $method;

    /** @var string */
    private $url;

    /** @var callable */
    private $complete_callback;

    /** @var callable */
    private $error_callback;

    /** @var resource */
    private $ch;

    /** @var resource */
    private $tmpfile;

    /** @var int */
    private $max_download_size;

    /**
     * Constructs cURL request object
     *
     * @param string $method
     * @param string $url
     * @param array $options
     */
    public function __construct($method, $url, array $options = [])
    {
        // method
        $this->method = $method;

        // URL
        $this->url = $url;
        if ( parse_url($url) === false ){
            throw new GrasshopperException('malformed url', Grasshopper::ERROR_MALFORMED_URL);
        }

        // method
        $this->max_download_size = isset($options['max_download_size']) ? $options['max_download_size'] : Grasshopper::DEFAULT_MAX_DOWNLOAD_SIZE;

        // callbacks
        $this->complete_callback = isset($options['complete']) ? $options['complete'] : null;
        $this->error_callback = isset($options['error']) ? $options['error'] : null;

        if ( $this->complete_callback && !is_callable($this->complete_callback) ){
            throw new GrasshopperException('invalid complete callback', Grasshopper::ERROR_INVALID_REQUEST_PARAMETER);
        }
        if ( $this->error_callback && !is_callable($this->error_callback) ){
            throw new GrasshopperException('invalid error callback', Grasshopper::ERROR_INVALID_REQUEST_PARAMETER);
        }

        $this->tmpfile = tmpfile();

        // options
        $dafaults = [
            /* user NOT customizable fields by options parameter */
            CURLOPT_URL => $url,
            CURLOPT_HEADER => true,
            CURLOPT_RETURNTRANSFER => false,

            /* user customizable fields by options parameter */
            CURLOPT_USERAGENT => Grasshopper::DEFAULT_USERAGENT,
            CURLOPT_PROXY => null,
            CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_NONE,
            CURLOPT_HTTPHEADER => (new CurlRequestHeader)->compile(),
            CURLOPT_TIMEOUT => 400,
            CURLOPT_CONNECTTIMEOUT => 0,
            CURLOPT_BUFFERSIZE => Grasshopper::DEFAULT_BUFFER_SIZE,

            /* SSL */
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_SSL_VERIFYHOST => false,

            /* HTTP/POST */
            CURLOPT_POST => false,
            CURLOPT_POSTFIELDS => '',

            /* redirection */
            CURLOPT_MAXREDIRS => 10,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_AUTOREFERER => true,

            /* file */
            CURLOPT_FILE => $this->tmpfile,
            CURLOPT_NOPROGRESS => false,
            CURLOPT_PROGRESSFUNCTION => function($down_size, $downloaded, $upload_size, $uploaded){
                return $downloaded > $this->max_download_size ? 1 : 0;
            },
        ];

        $user_curl_options = [
            /* user customizable fields by options parameter */
            CURLOPT_USERAGENT => isset($options['user_agent']) ? $options['user_agent'] : null,
            CURLOPT_PROXY => isset($options['proxy']) ? $options['proxy'] : null,
            CURLOPT_HTTP_VERSION => isset($options['http_version']) ? $options['http_version'] : null,
            CURLOPT_HTTPHEADER => isset($options['http_header']) ? $options['http_header'] : null,
            CURLOPT_TIMEOUT => isset($options['timeout']) ? $options['timeout'] : null,
            CURLOPT_CONNECTTIMEOUT => isset($options['connect_timeout']) ? $options['connect_timeout'] : null,
            CURLOPT_BUFFERSIZE => isset($options['buffer_size']) ? $options['buffer_size'] : null,

            /* HTTP/POST */
            CURLOPT_POST => ($this instanceof HttpPostRequest),
            CURLOPT_POSTFIELDS => isset($options['post_fields']) ? $options['post_fields'] : null,

            /* redirection */
            CURLOPT_MAXREDIRS => isset($options['max_redirs']) ? $options['max_redirs'] : null,
            CURLOPT_FOLLOWLOCATION => isset($options['follow_location']) ? $options['follow_location'] : null,
            CURLOPT_AUTOREFERER => isset($options['auto_referer']) ? $options['auto_referer'] : null,
        ];

        // HTTP header
        $user_http_header = isset($options['http_header']) ? $options['http_header'] : new CurlRequestHeader;
        if ( is_array($user_http_header) ){
            $user_http_header = new CurlRequestHeader($user_http_header);
        }
        elseif ( !($user_http_header instanceof CurlRequestHeader) ){
            throw new GrasshopperException('invalid http header', Grasshopper::ERROR_INVALID_REQUEST_PARAMETER);
        }
        $user_curl_options[CURLOPT_HTTPHEADER] = $user_http_header->compile();

        // merge options between user and defaults
        $real_options = [];
        foreach($dafaults as $k => $v){
            $user_val = isset($user_curl_options[$k]) ? $user_curl_options[$k] : null;
            $real_options[$k] = $user_val ? $user_val : $v;
        }

        // init curl handle
        $this->ch = curl_init();
        if ( !$this->ch ){
            throw new GrasshopperException('curl_init failed',Grasshopper::ERROR_INIT);
        }

        // set options to curl handle
        curl_setopt_array($this->ch, $real_options);
    }

    /**
     * Get URL
     */
    public function getUrl(){
        return $this->url;
    }

    /**
     * Get cURL handle
     */
    public function getCurlHandle(){
        return $this->ch;
    }

    /**
     * Get file handle
     */
    public function getFileHandle(){
        return $this->tmpfile;
    }

    /**
     * Get max download size
     */
    public function getMaxDownloadSize()
    {
        return $this->max_download_size;
    }

    /**
     * Detach from cURL multi handle
     *
     * @param resource $cm_handle
     */
    public function detach($cm_handle){
        if ( $this->ch ){
            curl_multi_remove_handle( $cm_handle, $this->ch );
        }
    }

    /**
     * close cURL handle and memory file handle
     */
    public function close(){
        if ( $this->ch ){
            curl_close($this->ch);
            $this->ch = null;
        }
    }

    /**
     * Callback when the request is successfully done.
     *
     * @param SuccessEvent $event
     */
    public function onRequestSucceeded(SuccessEvent $event)
    {
        if ( $this->complete_callback ){
            call_user_func_array( $this->complete_callback, [$event] );
        }
    }

    /**
     * Callback when the request has failed.
     *
     * @param ErrorEvent $event
     */
    public function onRequestFailed(ErrorEvent $event)
    {
        if ( $this->error_callback ){
            call_user_func_array( $this->error_callback, [$event] );
        }
    }

    /**
     * string conversion
     */
    public function __toString()
    {
        return 'CurlRequest(' . $this->url . ')';
    }
}<commit_msg>fixed http method problem added verbose/stderr options<commit_after><?php
namespace Grasshopper\curl;

use Grasshopper\Grasshopper;
use Grasshopper\exception\GrasshopperException;
use Grasshopper\HttpPostRequest;
use Grasshopper\event\SuccessEvent;
use Grasshopper\event\ErrorEvent;

class CurlRequest
{
    /** @var string */
    private $method;

    /** @var string */
    private $url;

    /** @var callable */
    private $complete_callback;

    /** @var callable */
    private $error_callback;

    /** @var resource */
    private $ch;

    /** @var resource */
    private $tmpfile;

    /** @var int */
    private $max_download_size;

    /**
     * Constructs cURL request object
     *
     * @param string $method
     * @param string $url
     * @param array $options
     */
    public function __construct($method, $url, array $options = [])
    {
        // method
        $this->method = strtoupper(trim($method));

        // URL
        $this->url = $url;
        if ( parse_url($url) === false ){
            throw new GrasshopperException('malformed url', Grasshopper::ERROR_MALFORMED_URL);
        }

        // method
        $this->max_download_size = isset($options['max_download_size']) ? $options['max_download_size'] : Grasshopper::DEFAULT_MAX_DOWNLOAD_SIZE;

        // callbacks
        $this->complete_callback = isset($options['complete']) ? $options['complete'] : null;
        $this->error_callback = isset($options['error']) ? $options['error'] : null;

        if ( $this->complete_callback && !is_callable($this->complete_callback) ){
            throw new GrasshopperException('invalid complete callback', Grasshopper::ERROR_INVALID_REQUEST_PARAMETER);
        }
        if ( $this->error_callback && !is_callable($this->error_callback) ){
            throw new GrasshopperException('invalid error callback', Grasshopper::ERROR_INVALID_REQUEST_PARAMETER);
        }

        $this->tmpfile = tmpfile();

        // options
        $dafaults = [
            /* user NOT customizable fields by options parameter */
            CURLOPT_URL => $url,
            CURLOPT_HEADER => true,
            CURLOPT_RETURNTRANSFER => false,

            /* user customizable fields by options parameter */
            CURLOPT_USERAGENT => Grasshopper::DEFAULT_USERAGENT,
            CURLOPT_PROXY => null,
            CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_NONE,
            CURLOPT_HTTPHEADER => (new CurlRequestHeader)->compile(),
            CURLOPT_TIMEOUT => 400,
            CURLOPT_CONNECTTIMEOUT => 0,
            CURLOPT_BUFFERSIZE => Grasshopper::DEFAULT_BUFFER_SIZE,

            /* SSL */
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_SSL_VERIFYHOST => false,

            /* HTTP/POST */
            CURLOPT_POSTFIELDS => '',

            /* redirection */
            CURLOPT_MAXREDIRS => 10,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_AUTOREFERER => true,

            /* file */
            CURLOPT_FILE => $this->tmpfile,
            CURLOPT_NOPROGRESS => false,
            CURLOPT_PROGRESSFUNCTION => function($down_size, $downloaded, $upload_size, $uploaded){
                return $downloaded > $this->max_download_size ? 1 : 0;
            },
        ];

        $user_curl_options = [
            /* user customizable fields by options parameter */
            CURLOPT_USERAGENT => isset($options['user_agent']) ? $options['user_agent'] : null,
            CURLOPT_PROXY => isset($options['proxy']) ? $options['proxy'] : null,
            CURLOPT_HTTP_VERSION => isset($options['http_version']) ? $options['http_version'] : null,
            CURLOPT_HTTPHEADER => isset($options['http_header']) ? $options['http_header'] : null,
            CURLOPT_TIMEOUT => isset($options['timeout']) ? $options['timeout'] : null,
            CURLOPT_CONNECTTIMEOUT => isset($options['connect_timeout']) ? $options['connect_timeout'] : null,
            CURLOPT_BUFFERSIZE => isset($options['buffer_size']) ? $options['buffer_size'] : null,

            /* HTTP/POST */
            CURLOPT_POST => ($this instanceof HttpPostRequest),
            CURLOPT_POSTFIELDS => isset($options['post_fields']) ? $options['post_fields'] : null,

            /* redirection */
            CURLOPT_MAXREDIRS => isset($options['max_redirs']) ? $options['max_redirs'] : null,
            CURLOPT_FOLLOWLOCATION => isset($options['follow_location']) ? $options['follow_location'] : null,
            CURLOPT_AUTOREFERER => isset($options['auto_referer']) ? $options['auto_referer'] : null,
        ];

        // HTTP header
        $user_http_header = isset($options['http_header']) ? $options['http_header'] : new CurlRequestHeader;
        if ( is_array($user_http_header) ){
            $user_http_header = new CurlRequestHeader($user_http_header);
        }
        elseif ( !($user_http_header instanceof CurlRequestHeader) ){
            throw new GrasshopperException('invalid http header', Grasshopper::ERROR_INVALID_REQUEST_PARAMETER);
        }
        $user_curl_options[CURLOPT_HTTPHEADER] = $user_http_header->compile();

        // merge options between user and defaults
        $real_options = [];
        foreach($dafaults as $k => $v){
            $user_val = isset($user_curl_options[$k]) ? $user_curl_options[$k] : null;
            $real_options[$k] = $user_val ? $user_val : $v;
        }

        // init curl handle
        $this->ch = curl_init();
        if ( !$this->ch ){
            throw new GrasshopperException('curl_init failed',Grasshopper::ERROR_INIT);
        }

        // set options to curl handle
        curl_setopt_array($this->ch, $real_options);

        curl_setopt($this->ch, CURLOPT_CUSTOMREQUEST, $this->method);
        curl_setopt($this->ch, CURLOPT_VERBOSE, isset($options['verbose']) ? $options['verbose'] : false);
        curl_setopt($this->ch, CURLOPT_STDERR, isset($options['stderr']) ? $options['stderr'] : STDERR);
    }

    /**
     * Get URL
     */
    public function getUrl(){
        return $this->url;
    }

    /**
     * Get cURL handle
     */
    public function getCurlHandle(){
        return $this->ch;
    }

    /**
     * Get file handle
     */
    public function getFileHandle(){
        return $this->tmpfile;
    }

    /**
     * Get max download size
     */
    public function getMaxDownloadSize()
    {
        return $this->max_download_size;
    }

    /**
     * Detach from cURL multi handle
     *
     * @param resource $cm_handle
     */
    public function detach($cm_handle){
        if ( $this->ch ){
            curl_multi_remove_handle( $cm_handle, $this->ch );
        }
    }

    /**
     * close cURL handle and memory file handle
     */
    public function close(){
        if ( $this->ch ){
            curl_close($this->ch);
            $this->ch = null;
        }
    }

    /**
     * Callback when the request is successfully done.
     *
     * @param SuccessEvent $event
     */
    public function onRequestSucceeded(SuccessEvent $event)
    {
        if ( $this->complete_callback ){
            call_user_func_array( $this->complete_callback, [$event] );
        }
    }

    /**
     * Callback when the request has failed.
     *
     * @param ErrorEvent $event
     */
    public function onRequestFailed(ErrorEvent $event)
    {
        if ( $this->error_callback ){
            call_user_func_array( $this->error_callback, [$event] );
        }
    }

    /**
     * string conversion
     */
    public function __toString()
    {
        return 'CurlRequest(' . $this->url . ')';
    }
}
{'fraction_non_alphanumeric': 0.1293039552590796, 'fraction_numerical': 0.001078094468027761, 'mean_word_length': 2.318131008271853, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 136, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5798093', 'n_tokens_mistral': 4448, 'n_tokens_neox': 4292, 'n_words': 823}