<commit_before><?php

class CM_Elasticsearch_Index_Cli extends CM_Cli_Runnable_Abstract {

    /**
     * @param string|null $indexName
     * @param bool|null   $skipIfExist
     */
    public function create($indexName = null, $skipIfExist = null) {
        if (null !== $indexName) {
            $indexes = array($this->_getIndex($indexName));
        } else {
            $indexes = $this->_getIndexes();
        }
        foreach ($indexes as $index) {
            if (!$index->getIndex()->exists() || !$skipIfExist) {
                $this->_getStreamOutput()->writeln('Creating elasticsearch index `' . $index->getIndex()->getName() . '`…');
                $index->createVersioned();
                $index->getIndex()->refresh();
            }
        }
    }

    /**
     * @param string|CM_Elasticsearch_Type_Abstract|null $indexName
     * @param string|null                                $host Elasticsearch host
     * @param int|null                                   $port Elasticsearch port
     * @throws CM_Exception_Invalid
     */
    public function update($indexName = null, $host = null, $port = null) {
        if ($indexName instanceof CM_Elasticsearch_Type_Abstract) {
            $indexes = array($indexName);
        } elseif (null !== $indexName) {
            $indexes = array($this->_getIndex($indexName, $host, $port));
        } else {
            $indexes = $this->_getIndexes($host, $port);
        }
        foreach ($indexes as $index) {
            $this->_getStreamOutput()->writeln('Updating elasticsearch index `' . $index->getIndex()->getName() . '`...');
            $indexName = $index->getIndex()->getName();
            $key = 'Search.Updates_' . $index->getType()->getName();
            try {
                $ids = $this->_getRedis()->sFlush($key);
                $ids = array_filter(array_unique($ids));
                $index->update($ids);
                $index->getIndex()->refresh();
            } catch (Exception $e) {
                $message = $indexName . '-updates failed.' . PHP_EOL;
                if (isset($ids)) {
                    $message .= 'Re-adding ' . count($ids) . ' ids to queue.' . PHP_EOL;
                    foreach ($ids as $id) {
                        $this->_getRedis()->sAdd($key, $id);
                    }
                }
                $message .= 'Reason: ' . $e->getMessage() . PHP_EOL;
                throw new CM_Exception_Invalid($message);
            }
        }
    }

    /**
     * @param string|null $indexName
     */
    public function delete($indexName = null) {
        if (null !== $indexName) {
            $indexes = array($this->_getIndex($indexName));
        } else {
            $indexes = $this->_getIndexes();
        }
        foreach ($indexes as $index) {
            if ($index->getIndex()->exists()) {
                $this->_getStreamOutput()->writeln('Deleting elasticsearch index `' . $index->getIndex()->getName() . '`…');
                $index->getIndex()->delete();
            }
        }
    }

    public function optimize() {
        $servers = CM_Config::get()->CM_Elasticsearch_Client->servers;
        foreach ($servers as $server) {
            $client = new Elastica\Client($server);
            $client->optimizeAll();
        }
    }<commit_msg>Use clients instead of outdated configs<commit_after><?php

class CM_Elasticsearch_Index_Cli extends CM_Cli_Runnable_Abstract {

    /**
     * @param string|null $indexName
     * @param bool|null   $skipIfExist
     */
    public function create($indexName = null, $skipIfExist = null) {
        if (null !== $indexName) {
            $indexes = array($this->_getIndex($indexName));
        } else {
            $indexes = $this->_getIndexes();
        }
        foreach ($indexes as $index) {
            if (!$index->getIndex()->exists() || !$skipIfExist) {
                $this->_getStreamOutput()->writeln('Creating elasticsearch index `' . $index->getIndex()->getName() . '`…');
                $index->createVersioned();
                $index->getIndex()->refresh();
            }
        }
    }

    /**
     * @param string|CM_Elasticsearch_Type_Abstract|null $indexName
     * @param string|null                                $host Elasticsearch host
     * @param int|null                                   $port Elasticsearch port
     * @throws CM_Exception_Invalid
     */
    public function update($indexName = null, $host = null, $port = null) {
        if ($indexName instanceof CM_Elasticsearch_Type_Abstract) {
            $indexes = array($indexName);
        } elseif (null !== $indexName) {
            $indexes = array($this->_getIndex($indexName, $host, $port));
        } else {
            $indexes = $this->_getIndexes($host, $port);
        }
        foreach ($indexes as $index) {
            $this->_getStreamOutput()->writeln('Updating elasticsearch index `' . $index->getIndex()->getName() . '`...');
            $indexName = $index->getIndex()->getName();
            $key = 'Search.Updates_' . $index->getType()->getName();
            try {
                $ids = $this->_getRedis()->sFlush($key);
                $ids = array_filter(array_unique($ids));
                $index->update($ids);
                $index->getIndex()->refresh();
            } catch (Exception $e) {
                $message = $indexName . '-updates failed.' . PHP_EOL;
                if (isset($ids)) {
                    $message .= 'Re-adding ' . count($ids) . ' ids to queue.' . PHP_EOL;
                    foreach ($ids as $id) {
                        $this->_getRedis()->sAdd($key, $id);
                    }
                }
                $message .= 'Reason: ' . $e->getMessage() . PHP_EOL;
                throw new CM_Exception_Invalid($message);
            }
        }
    }

    /**
     * @param string|null $indexName
     */
    public function delete($indexName = null) {
        if (null !== $indexName) {
            $indexes = array($this->_getIndex($indexName));
        } else {
            $indexes = $this->_getIndexes();
        }
        foreach ($indexes as $index) {
            if ($index->getIndex()->exists()) {
                $this->_getStreamOutput()->writeln('Deleting elasticsearch index `' . $index->getIndex()->getName() . '`…');
                $index->getIndex()->delete();
            }
        }
    }

    public function optimize() {
        foreach (CM_Service_Manager::getInstance()->getElasticsearch()->getClients() as $client) {
            $client->optimizeAll();
        }
    }
{'fraction_non_alphanumeric': 0.15714722052535124, 'fraction_numerical': 0.0, 'mean_word_length': 1.7586352148272957, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1544368', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1791, 'n_words': 338}