<commit_before><?php

use Guzzle\Http\Client;
use Guzzle\Http\Exception\RequestException;

class TikaRestClient extends Client
{
    /**
     * Detect if the service is available
     *
     * @return bool
     */
    public function isAvailable()
    {
        try {
            return $this
                ->get()->send()
                ->getStatusCode() == 200;
        } catch (RequestException $ex) {
            return false;
        }
    }

    /**
     * Get version code
     *
     * @return float
     */
    public function getVersion()
    {
        $response = $this->get('version')->send();
        // Parse output
        if ($response->getStatusCode() == 200 &&
            preg_match('/Apache Tika (?<version>[\.\d]+)/', $response->getBody(), $matches)
        ) {
            return (float)$matches['version'];
        }

        return 0.0;
    }

    protected $mimes = array();

    /**
     * Gets supported mime data. May include aliased mime types.
     *
     * @return array
     */
    public function getSupportedMimes()
    {
        if ($this->mimes) {
            return $this->mimes;
        }

        $response = $this->get(
            'mime-types',
            array('Accept' => 'application/json')
        )->send();

        return $this->mimes = $response->json();
    }

    /**
     * Extract text content from a given file.
     * Logs a notice-level error if the document can't be parsed.
     *
     * @param string $file Full filesystem path to a file to post
     * @return string Content of the file extracted as plain text
     */
    public function tika($file)
    {
        $text = null;
        try {
            $response = $this->put(
                'tika',
                array('Accept' => 'text/plain'),
                file_get_contents($file)
            )->send();
            $text = $response->getBody(true);
        } catch (RequestException $e) {
            $msg = sprintf(
                'TikaRestClient was not able to process %s. Response: %s %s.',
                $file,
                $e->getResponse()->getStatusCode(),
                $e->getResponse()->getReasonPhrase()
            );

            // Only available if tika-server was started with --includeStack
            $body = $e->getResponse()->getBody(true);
            if ($body) {
                $msg .= ' Body: ' . $body;
            }

            SS_Log::log($msg, SS_Log::NOTICE);
        }
        
        return $text;
    }
}
<commit_msg>Allow username and password in requests to Tika server (#35)<commit_after><?php

use Guzzle\Http\Client;
use Guzzle\Http\Exception\RequestException;

class TikaRestClient extends Client
{
    /**
     * Authentication options to be sent to the Tika server
     *
     * @var array
     */
    protected $options = array('username' => null, 'password' => null);

    /**
     * @var array
     */
    protected $mimes = array();

    public function __construct($baseUrl = '', $config = null)
    {
        if (defined('SS_TIKA_USERNAME') && defined('SS_TIKA_PASSWORD')) {
            $this->options = array(
                'username' => SS_TIKA_USERNAME,
                'password' => <PASSWORD>,
            );
        }
        parent::__construct($baseUrl, $config);
    }

    /**
     * Detect if the service is available
     *
     * @return bool
     */
    public function isAvailable()
    {
        try {
            $result = $this->get(null);
            $result->setAuth($this->options['username'], $this->options['password']);
            $result->send();
            if ($result->getResponse()->getStatusCode() == 200) {
                return true;
            }
        } catch (RequestException $ex) {
            SS_Log::log(sprintf("Tika unavailable - %s", $ex->getMessage()), SS_Log::ERR);
            return false;
        }
    }

    /**
     * Get version code
     *
     * @return float
     */
    public function getVersion()
    {
        $response = $this->get('version');
        $response->setAuth($this->options['username'], $this->options['password']);
        $response->send();
        $version = 0.0;
        // Parse output
        if ($response->getResponse()->getStatusCode() == 200 &&
            preg_match('/Apache Tika (?<version>[\.\d]+)/', $response->getResponse()->getBody(), $matches)
        ) {
            $version = (float)$matches['version'];
        }
        return $version;
    }

    /**
     * Gets supported mime data. May include aliased mime types.
     *
     * @return array
     */
    public function getSupportedMimes()
    {
        if ($this->mimes) {
            return $this->mimes;
        }
        $response = $this->get(
            'mime-types',
            array('Accept' => 'application/json')
        );
        $response->setAuth($this->options['username'], $this->options['password']);
        $response->send();
        return $this->mimes = $response->getResponse()->json();
    }

    /**
     * Extract text content from a given file.
     * Logs a notice-level error if the document can't be parsed.
     *
     * @param string $file Full filesystem path to a file to post
     * @return string Content of the file extracted as plain text
     */
    public function tika($file)
    {
        $text = null;
        try {
            $response = $this->put(
                'tika',
                array('Accept' => 'text/plain'),
                file_get_contents($file)
            );
            $response->setAuth($this->options['username'], $this->options['password']);
            $response->send();
            $text = $response->getResponse()->getBody(true);
        } catch (RequestException $e) {
            $msg = sprintf(
                'TikaRestClient was not able to process %s. Response: %s %s.',
                $file,
                $e->getResponse()->getStatusCode(),
                $e->getResponse()->getReasonPhrase()
            );
            // Only available if tika-server was started with --includeStack
            $body = $e->getResponse()->getBody(true);
            if ($body) {
                $msg .= ' Body: ' . $body;
            }
            SS_Log::log($msg, SS_Log::NOTICE);
        }
        return $text;
    }
}

{'fraction_non_alphanumeric': 0.14581328200192492, 'fraction_numerical': 0.0028873917228103944, 'mean_word_length': 1.661118224498506, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '332500', 'n_tokens_mistral': 1766, 'n_tokens_neox': 1704, 'n_words': 413}