<commit_before>#!/usr/bin/env php
<?php
/**
 * @package     Devvoh Fluid Installer
 * @license     MIT
 * @author      <NAME> <<EMAIL>>
 * @copyright   2015-2016, <NAME>, devvoh webdevelopment
 */

class Curl {

    /**
     * @var string
     */
    protected $userAgent = 'fluid/curl';

    /**
     * Return the user agent
     *
     * @return string
     */
    public function getUserAgent() {
        return $this->userAgent;
    }

    /**
     * Sets the user agent
     *
     * @param $userAgent
     *
     * @return $this
     */
    public function setUserAgent($userAgent) {
        $this->userAgent = $userAgent;
        return $this;
    }

    /**
     * Returns the result from loading url
     *
     * @param string $url
     *
     * @return string|false
     */
    public function getContent($url = null) {
        if (!$url) {
            return false;
        }

        $ch = curl_init($url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
//        curl_setopt($ch, CURLOPT_USERPWD, '<PASSWORD>:<PASSWORD>');
        curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent());
        $result = curl_exec($ch);
        curl_close($ch);
        return $result;
    }

    /**
     * Downloads $url to $filename
     *
     * @param string $url
     * @param string $path
     * @param string $filename
     *
     * @return string|false
     */
    public function download($url = null, $path = null, $filename = null) {
        if (!$url || !$path || !$filename) {
            return false;
        }
        if (!is_dir($path) || !is_writable($path)) {
            return false;
        }
        $filename = $path . DIRECTORY_SEPARATOR . $filename;
        $fp = fopen($filename, 'w+');

        $ch = curl_init($url);
        curl_setopt($ch, CURLOPT_FILE, $fp);
//        curl_setopt($ch, CURLOPT_USERPWD, '<PASSWORD>:<PASSWORD>');
        curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent());
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_exec($ch);<commit_msg>Removed inactive token<commit_after>#!/usr/bin/env php
<?php
/**
 * @package     Devvoh Fluid Installer
 * @license     MIT
 * @author      <NAME> <<EMAIL>>
 * @copyright   2015-2016, <NAME>, devvoh webdevelopment
 */

class Curl {

    /**
     * @var string
     */
    protected $userAgent = 'fluid/curl';

    /**
     * Return the user agent
     *
     * @return string
     */
    public function getUserAgent() {
        return $this->userAgent;
    }

    /**
     * Sets the user agent
     *
     * @param $userAgent
     *
     * @return $this
     */
    public function setUserAgent($userAgent) {
        $this->userAgent = $userAgent;
        return $this;
    }

    /**
     * Returns the result from loading url
     *
     * @param string $url
     *
     * @return string|false
     */
    public function getContent($url = null) {
        if (!$url) {
            return false;
        }

        $ch = curl_init($url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent());
        $result = curl_exec($ch);
        curl_close($ch);
        return $result;
    }

    /**
     * Downloads $url to $filename
     *
     * @param string $url
     * @param string $path
     * @param string $filename
     *
     * @return string|false
     */
    public function download($url = null, $path = null, $filename = null) {
        if (!$url || !$path || !$filename) {
            return false;
        }
        if (!is_dir($path) || !is_writable($path)) {
            return false;
        }
        $filename = $path . DIRECTORY_SEPARATOR . $filename;
        $fp = fopen($filename, 'w+');

        $ch = curl_init($url);
        curl_setopt($ch, CURLOPT_FILE, $fp);
        curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent());
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_exec($ch);
{'fraction_non_alphanumeric': 0.14391050088990592, 'fraction_numerical': 0.004068141367912535, 'mean_word_length': 1.9780469341408025, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5588420', 'n_tokens_mistral': 1301, 'n_tokens_neox': 1241, 'n_words': 277}