<commit_before><?php

namespace neTpyceB\TMCms\Orm;

use neTpyceB\TMCms\Cache\Cacher;
use neTpyceB\TMCms\DB\SQL;
use neTpyceB\TMCms\Files\FileSystem;
use neTpyceB\TMCms\Strings\Converter;

class EntityRepository {
    protected $db_table = ''; // Should be overwritten in extended class
    protected $translation_fields = []; // Should be overwritten in extended class

    private static $_cache_key_prefix = 'orm_entity_repository_';

    private $sql_where_fields = [];
    private $sql_select_fields = [];
    private $sql_offset = 0;
    private $sql_limit = 0;
    private $order_fields = [];
    private $order_random = false;
    private $group_by_fields = [];
    private $having_fields = [];
    private $translation_joins = [];

    private $use_iterator = true;
    private $collected_objects = [];
    private $collected_objects_data = [];

    private $total_count_rows;
    private $require_to_count_total_rows = false;

    protected $debug = false;
    private $use_cache = false;
    private $cache_ttl = 60;

    private $join_tables = [];
    private $last_used_sql;

    public function __construct($ids = []) {
        if ($ids) {
            $this->setIds($ids);
        }

        return $this;
    }

    public function deleteObjectCollection() {
        $this->collectObjects();

        // Call delete on every object
        foreach ($this->getCollectedObjects() as $v) {
            /** @var Entity $v */
            $v->deleteObject();
        }

        return $this;
    }

    /**
     * @return array
     */
    public function getDbTableFields()
    {
        return SQL::getFields($this->getDbTableName());
    }

    /**
     * @param bool $skip_objects_creation - true if no need to create objects
     * @param bool $skip_changed_fields - skip update of changed fields
     * @return $this
     */
    protected function collectObjects($skip_objects_creation = false, $skip_changed_fields = false)
    {
        $sql = $this->getSelectSql();
        if ($sql === $this->last_used_sql) {
            // Skip queries - nothing changed
            return $this;
        }
        $this->last_used_sql = $sql;

        // Check cache for this exact collection
        if ($this->use_cache) {
            //Check cached values, set local properties
            $data = Cacher::getInstance()->getDefaultCacher()->get($this->getCacheKey($sql));
            if ($data && is_array($data) && isset($data['collection_arrays'], $data['collection_objects'])) {
                // Set local data
                $this->collected_objects_data = $data['collected_objects_data'];
                $this->collected_objects = $data['collected_objects'];

                // No further actions
                return $this;
            }
        }

            // Use Iterator in DB query
        if ($this->use_iterator) {
            $this->collected_objects_data = SQL::q_assoc_iterator($sql, false);
        } else {
            $this->collected_objects_data = SQL::q_assoc($sql, false);
        }

        if ($this->require_to_count_total_rows) {
            $this->total_count_rows = q_value('SELECT FOUND_ROWS();');
        }

        $this->collected_objects = []; // Reset objects

        if (!$skip_objects_creation) {
            // Need to create objects from array data
            foreach ($this->collected_objects_data as $v) {
                $class = $this->getObjectClass();
                /** @var Entity $obj */
                $obj = new $class();

                if (!isset($v['id'])) {
                    SQL::getInstance()->addPrimaryAutoIncrementIdFieldToTable($this->getDbTableName());
                    dump('No ID field found for ' . get_class($this) . '. Field "id" with auto-increment was created in table "'. $this->getDbTableName() .'". Please reload page.');
                }

                // Prevent auto-query db, skip tables with no id field
                $id = $v['id'];
                unset($v['id']);

                // Set object data
                $obj->loadDataFromArray($v, $skip_changed_fields);

                // Set current ID
                $obj->setId($id, false);

                // Save in returning array ob objects
                $this->collected_objects[] = $obj;
            }
        }

        if ($this->use_cache) {
            // Save all collected data to Cache
            $data = [
                'collected_objects_data' => $this->collected_objects_data,
                'collected_objects' => $this->collected_objects
            ];
            Cacher::getInstance()->getDefaultCacher()->set($this->getCacheKey($sql), $data, $this->cache_ttl);
        }

        return $this;
    }

    protected function getCollectedObjects()
    {
        return $this->collected_objects;
    }

    /**
     * Set collected objects in Repository - may be useful in mass-updates
     * @param array $objects
     * @return $this
     */
    public function setCollectedObjects(array $objects)
    {
        $this->collected_objects = $objects;

        return $this;
    }

    protected function getCollectedData()
    {
        return $this->collected_objects_data;
    }

    /**
     * @param array $ids
     * @return $this
     */
    public function setIds(array $ids)
    {
        $this->setFilterValueWhereIn('id', $ids);

        return $this;
    }

    public function getIds() {
        $ids = [];

        foreach ($this->getAsArrayOfObjectData() as $v) {
            $ids[] = $v['id'];
        }

        return $ids;
    }

    public function getSumOfOneField($field) {
        $sum = 0;

        foreach ($this->getAsArrayOfObjectData() as $v) {
            $sum += $v[$field];
        }

        return $sum;
    }

    public function addGroupBy($field, $table = '') {
        // No table provided
        if (!$table) {
            $table = $this->getDbTableName();
        }

        $this->group_by_fields[] = [
            'table' => $table,
            'field' => $field
        ];

        return $this;
    }

    public function addHaving($field, $value) {
        $this->having_fields[] = [
            'field' => $field,
            'value' => $value
        ];
    }

    public function flipBoolValue($field) {
        if (!$this->getCollectedObjects()) {
            $this->collectObjects(false, true);
        }

        foreach ($this->getCollectedObjects() as $object) {
            /** @var Entity $object */
            $object->flipBoolValue($field);
        }

        return $this;
    }

    public function save()
    {
        if (!$this->getCollectedObjects()) {
            $this->collectObjects(false, true);
        }

        foreach ($this->getCollectedObjects() as $object) {
            /** @var Entity $object */
            $object->save();
        }

        return $this;
    }

    /**
     * @return bool
     */
    public function hasAnyObjectInCollection()
    {
        $this->collectObjects(true);

        $obj = $this->getFirstObjectFromCollection();

        return (bool)$obj;
    }

    /**
     * @param int $count
     * @return bool
     */
    public function hasExactCountOfObjects($count)
    {
        $this->collectObjects(true);

        return (bool)((int)$this->getCountOfObjectsInCollection() === (int)$count);
    }

    /**
     * @return bool
     */
    public function getCountOfObjectsInCollection()
    {
        $this->collectObjects(true);

        return count($this->collected_objects_data);
    }

    /**
     * @return bool
     */
    public function getCountOfMaxPossibleFoundObjectsWithoutFilters()
    {
        return (int)q_value($this->getSelectSql(true));
    }

    /**
     * @return Entity
     */
    public function getFirstObjectFromCollection()
    {
        $limit_tmp = $this->sql_limit;
        $this->setLimit(1);

        foreach ($this->getAsArrayOfObjectData() as $obj_data) {
            $this->setLimit($limit_tmp);

            $class = $this->getObjectClass();
            /** @var Entity $obj */
            $obj = new $class();
            $obj->loadDataFromArray($obj_data, true);

            return $obj;
        }

        $this->setLimit($limit_tmp);
        return NULL;
    }

    /**
     * @return Entity
     */
    public function getLastObjectFromCollection()
    {
        $objects = $this->getAsArrayOfObjects();
        if ($objects) {
            return array_pop($objects);
        }

        return NULL;
    }

    public function getAsArrayOfObjects()
    {
        $this->collectObjects();

        return $this->getCollectedObjects();
    }

    /**
     * @param bool $non_iterator - do not use Iterator, may be usefull for dumping output
     * @return array
     */
    public function getAsArrayOfObjectData($non_iterator = false)
    {
        if ($non_iterator) {
            $this->setGenerateOutputWithIterator(false);
        }

        $this->collectObjects(true);

        return $this->getCollectedData();
    }

    /**
     * @param string $value_field
     * @param string $key_field
     * @return array
     */
    public function getPairs($value_field, $key_field = '')
    {
        if (!$key_field) {
            $key_field = 'id';
        }

        $this->collectObjects();

        $pairs = [];
        foreach ($this->getAsArrayOfObjects() as $v) {
            /** @var Entity $v */
            $v->loadDataFromDB();

            $key_method = 'get' . ucfirst($key_field);
            $value_method = 'get' . ucfirst($value_field);
            $pairs[$v->{$key_method}()] = $v->{$value_method}();
        }

        return $pairs;
    }

    /**
     * @param int $limit
     * @return $this
     */
    public function setLimit($limit) {
        $this->sql_limit = (int)$limit;

        return $this;
    }

    /**
     * @param int $offset
     * @return $this
     */
    public function setOffset($offset) {
        $this->sql_offset = (int)$offset;

        return $this;
    }

    /**
     * @param string $field
     * @param bool $direction_desc
     * @param string $table
     * @param bool $do_not_use_table_in_sql required in some conditions with temp fields
     * @return $this
     */
    public function addOrderByField($field, $direction_desc = false, $table = '', $do_not_use_table_in_sql = false) {
        // No table provided
        if (!$table) {
            $table = $this->getDbTableName();
        }

        $direction = $direction_desc ? 'DESC' : ' ASC';

        if (in_array($field, $this->translation_fields)) {
            $k = count($this->translation_joins);
            $this->translation_fields[] = 'LEFT JOIN `cms_translations` AS `d' . $k . '` ON (`d' . $k . '`.`id` = '. $table .'.`' . $field . '`)';
            $this->order_fields[] = [
                'table' => false,
                'field' => '`d' . $k . '`.`' . LNG . '`',
                'direction' => $direction,
                'do_not_use_table_in_sql' => true,
                'type' => 'string'
            ];
        } else {
            $this->order_fields[] = [
                'table' => $table,
                'field' => $field,
                'direction' => $direction,
                'do_not_use_table_in_sql' => $do_not_use_table_in_sql,
                'type' => 'simple'
            ];
        }

        return $this;
    }

    /**
     * @param bool
     * @return $this
     */
    public function setOrderByRandom($flag) {
        $this->order_random = $flag;

        return $this;
    }

    public function clearCollectionCache() {
        $this->last_used_sql = '';
        $this->collectObjects();

        return $this;
    }

    public function addSimpleSelectFields(array $fields, $table = false) {
        if (!$table) {
            $table = $this->getDbTableName();
        }

        foreach ($fields as $k => $field) {
            $this->sql_select_fields[] = [
                'table' => $table,
                'field' => $field,
                'as' => false,
                'type' => 'simple'
            ];
        }

        return $this;
    }

    public function addSelectFieldAsAlias($field, $alias, $table = false) {
        if (!$table) {
            $table = $this->getDbTableName();
        }

        $this->sql_select_fields[] = [
            'table' => $table,
            'field' => $field,
            'as' => $alias,
            'type' => 'simple'
        ];

        return $this;
    }

    public function addSelectFieldAsString($sql) {
        $this->sql_select_fields[] = [
            'table' => false,
            'field' => $sql,
            'as' => false,
            'type' => 'string'
        ];

        return $this;
    }

    public function getSelectFields() {
        return $this->sql_select_fields;
    }

    public function getTotalSelectedRowsWithoutLimit() {
        return $this->total_count_rows;
    }
<commit_msg>EntityRepository - new method for SELECT COUNT(*) in inner select<commit_after><?php

namespace neTpyceB\TMCms\Orm;

use neTpyceB\TMCms\Cache\Cacher;
use neTpyceB\TMCms\DB\SQL;
use neTpyceB\TMCms\Files\FileSystem;
use neTpyceB\TMCms\Strings\Converter;

class EntityRepository {
    protected $db_table = ''; // Should be overwritten in extended class
    protected $translation_fields = []; // Should be overwritten in extended class

    private static $_cache_key_prefix = 'orm_entity_repository_';

    private $sql_where_fields = [];
    private $sql_select_fields = [];
    private $sql_offset = 0;
    private $sql_limit = 0;
    private $order_fields = [];
    private $order_random = false;
    private $group_by_fields = [];
    private $having_fields = [];
    private $translation_joins = [];

    private $use_iterator = true;
    private $collected_objects = [];
    private $collected_objects_data = [];

    private $total_count_rows;
    private $require_to_count_total_rows = false;

    protected $debug = false;
    private $use_cache = false;
    private $cache_ttl = 60;

    private $join_tables = [];
    private $last_used_sql;

    public function __construct($ids = []) {
        if ($ids) {
            $this->setIds($ids);
        }

        return $this;
    }

    public function deleteObjectCollection() {
        $this->collectObjects();

        // Call delete on every object
        foreach ($this->getCollectedObjects() as $v) {
            /** @var Entity $v */
            $v->deleteObject();
        }

        return $this;
    }

    /**
     * @return array
     */
    public function getDbTableFields()
    {
        return SQL::getFields($this->getDbTableName());
    }

    /**
     * @param bool $skip_objects_creation - true if no need to create objects
     * @param bool $skip_changed_fields - skip update of changed fields
     * @return $this
     */
    protected function collectObjects($skip_objects_creation = false, $skip_changed_fields = false)
    {
        $sql = $this->getSelectSql();
        if ($sql === $this->last_used_sql) {
            // Skip queries - nothing changed
            return $this;
        }
        $this->last_used_sql = $sql;

        // Check cache for this exact collection
        if ($this->use_cache) {
            //Check cached values, set local properties
            $data = Cacher::getInstance()->getDefaultCacher()->get($this->getCacheKey($sql));
            if ($data && is_array($data) && isset($data['collection_arrays'], $data['collection_objects'])) {
                // Set local data
                $this->collected_objects_data = $data['collected_objects_data'];
                $this->collected_objects = $data['collected_objects'];

                // No further actions
                return $this;
            }
        }

            // Use Iterator in DB query
        if ($this->use_iterator) {
            $this->collected_objects_data = SQL::q_assoc_iterator($sql, false);
        } else {
            $this->collected_objects_data = SQL::q_assoc($sql, false);
        }

        if ($this->require_to_count_total_rows) {
            $this->total_count_rows = q_value('SELECT FOUND_ROWS();');
        }

        $this->collected_objects = []; // Reset objects

        if (!$skip_objects_creation) {
            // Need to create objects from array data
            foreach ($this->collected_objects_data as $v) {
                $class = $this->getObjectClass();
                /** @var Entity $obj */
                $obj = new $class();

                if (!isset($v['id'])) {
                    SQL::getInstance()->addPrimaryAutoIncrementIdFieldToTable($this->getDbTableName());
                    dump('No ID field found for ' . get_class($this) . '. Field "id" with auto-increment was created in table "'. $this->getDbTableName() .'". Please reload page.');
                }

                // Prevent auto-query db, skip tables with no id field
                $id = $v['id'];
                unset($v['id']);

                // Set object data
                $obj->loadDataFromArray($v, $skip_changed_fields);

                // Set current ID
                $obj->setId($id, false);

                // Save in returning array ob objects
                $this->collected_objects[] = $obj;
            }
        }

        if ($this->use_cache) {
            // Save all collected data to Cache
            $data = [
                'collected_objects_data' => $this->collected_objects_data,
                'collected_objects' => $this->collected_objects
            ];
            Cacher::getInstance()->getDefaultCacher()->set($this->getCacheKey($sql), $data, $this->cache_ttl);
        }

        return $this;
    }

    protected function getCollectedObjects()
    {
        return $this->collected_objects;
    }

    /**
     * Set collected objects in Repository - may be useful in mass-updates
     * @param array $objects
     * @return $this
     */
    public function setCollectedObjects(array $objects)
    {
        $this->collected_objects = $objects;

        return $this;
    }

    protected function getCollectedData()
    {
        return $this->collected_objects_data;
    }

    /**
     * @param array $ids
     * @return $this
     */
    public function setIds(array $ids)
    {
        $this->setFilterValueWhereIn('id', $ids);

        return $this;
    }

    public function getIds() {
        $ids = [];

        foreach ($this->getAsArrayOfObjectData() as $v) {
            $ids[] = $v['id'];
        }

        return $ids;
    }

    public function getSumOfOneField($field) {
        $sum = 0;

        foreach ($this->getAsArrayOfObjectData() as $v) {
            $sum += $v[$field];
        }

        return $sum;
    }

    public function addGroupBy($field, $table = '') {
        // No table provided
        if (!$table) {
            $table = $this->getDbTableName();
        }

        $this->group_by_fields[] = [
            'table' => $table,
            'field' => $field
        ];

        return $this;
    }

    public function addHaving($field, $value) {
        $this->having_fields[] = [
            'field' => $field,
            'value' => $value
        ];
    }

    public function flipBoolValue($field) {
        if (!$this->getCollectedObjects()) {
            $this->collectObjects(false, true);
        }

        foreach ($this->getCollectedObjects() as $object) {
            /** @var Entity $object */
            $object->flipBoolValue($field);
        }

        return $this;
    }

    public function save()
    {
        if (!$this->getCollectedObjects()) {
            $this->collectObjects(false, true);
        }

        foreach ($this->getCollectedObjects() as $object) {
            /** @var Entity $object */
            $object->save();
        }

        return $this;
    }

    /**
     * @return bool
     */
    public function hasAnyObjectInCollection()
    {
        $this->collectObjects(true);

        $obj = $this->getFirstObjectFromCollection();

        return (bool)$obj;
    }

    /**
     * @param int $count
     * @return bool
     */
    public function hasExactCountOfObjects($count)
    {
        $this->collectObjects(true);

        return (bool)((int)$this->getCountOfObjectsInCollection() === (int)$count);
    }

    /**
     * @return bool
     */
    public function getCountOfObjectsInCollection()
    {
        $this->collectObjects(true);

        return count($this->collected_objects_data);
    }

    /**
     * @return bool
     */
    public function getCountOfMaxPossibleFoundObjectsWithoutFilters()
    {
        return (int)q_value($this->getSelectSql(true));
    }

    /**
     * @return Entity
     */
    public function getFirstObjectFromCollection()
    {
        $limit_tmp = $this->sql_limit;
        $this->setLimit(1);

        foreach ($this->getAsArrayOfObjectData() as $obj_data) {
            $this->setLimit($limit_tmp);

            $class = $this->getObjectClass();
            /** @var Entity $obj */
            $obj = new $class();
            $obj->loadDataFromArray($obj_data, true);

            return $obj;
        }

        $this->setLimit($limit_tmp);
        return NULL;
    }

    /**
     * @return Entity
     */
    public function getLastObjectFromCollection()
    {
        $objects = $this->getAsArrayOfObjects();
        if ($objects) {
            return array_pop($objects);
        }

        return NULL;
    }

    public function getAsArrayOfObjects()
    {
        $this->collectObjects();

        return $this->getCollectedObjects();
    }

    /**
     * @param bool $non_iterator - do not use Iterator, may be usefull for dumping output
     * @return array
     */
    public function getAsArrayOfObjectData($non_iterator = false)
    {
        if ($non_iterator) {
            $this->setGenerateOutputWithIterator(false);
        }

        $this->collectObjects(true);

        return $this->getCollectedData();
    }

    /**
     * @param string $value_field
     * @param string $key_field
     * @return array
     */
    public function getPairs($value_field, $key_field = '')
    {
        if (!$key_field) {
            $key_field = 'id';
        }

        $this->collectObjects();

        $pairs = [];
        foreach ($this->getAsArrayOfObjects() as $v) {
            /** @var Entity $v */
            $v->loadDataFromDB();

            $key_method = 'get' . ucfirst($key_field);
            $value_method = 'get' . ucfirst($value_field);
            $pairs[$v->{$key_method}()] = $v->{$value_method}();
        }

        return $pairs;
    }

    /**
     * @param int $limit
     * @return $this
     */
    public function setLimit($limit) {
        $this->sql_limit = (int)$limit;

        return $this;
    }

    /**
     * @param int $offset
     * @return $this
     */
    public function setOffset($offset) {
        $this->sql_offset = (int)$offset;

        return $this;
    }

    /**
     * @param string $field
     * @param bool $direction_desc
     * @param string $table
     * @param bool $do_not_use_table_in_sql required in some conditions with temp fields
     * @return $this
     */
    public function addOrderByField($field, $direction_desc = false, $table = '', $do_not_use_table_in_sql = false) {
        // No table provided
        if (!$table) {
            $table = $this->getDbTableName();
        }

        $direction = $direction_desc ? 'DESC' : ' ASC';

        if (in_array($field, $this->translation_fields)) {
            $k = count($this->translation_joins);
            $this->translation_fields[] = 'LEFT JOIN `cms_translations` AS `d' . $k . '` ON (`d' . $k . '`.`id` = '. $table .'.`' . $field . '`)';
            $this->order_fields[] = [
                'table' => false,
                'field' => '`d' . $k . '`.`' . LNG . '`',
                'direction' => $direction,
                'do_not_use_table_in_sql' => true,
                'type' => 'string'
            ];
        } else {
            $this->order_fields[] = [
                'table' => $table,
                'field' => $field,
                'direction' => $direction,
                'do_not_use_table_in_sql' => $do_not_use_table_in_sql,
                'type' => 'simple'
            ];
        }

        return $this;
    }

    /**
     * @param bool
     * @return $this
     */
    public function setOrderByRandom($flag) {
        $this->order_random = $flag;

        return $this;
    }

    public function clearCollectionCache() {
        $this->last_used_sql = '';
        $this->collectObjects();

        return $this;
    }

    public function addSimpleSelectFields(array $fields, $table = false) {
        if (!$table) {
            $table = $this->getDbTableName();
        }

        foreach ($fields as $k => $field) {
            $this->sql_select_fields[] = [
                'table' => $table,
                'field' => $field,
                'as' => false,
                'type' => 'simple'
            ];
        }

        return $this;
    }

    public function addSelectFieldAsAlias($field, $alias, $table = false) {
        if (!$table) {
            $table = $this->getDbTableName();
        }

        $this->sql_select_fields[] = [
            'table' => $table,
            'field' => $field,
            'as' => $alias,
            'type' => 'simple'
        ];

        return $this;
    }

    public function addSelectFieldAsString($sql) {
        $this->sql_select_fields[] = [
            'table' => false,
            'field' => $sql,
            'as' => false,
            'type' => 'string'
        ];

        return $this;
    }

    public function addSelectCountFromPairedObject(EntityRepository $repository, $field_name, $count_by_field) {
        $this->sql_select_fields[] = [
            'table' => false,
            'field' => '(SELECT COUNT(*) FROM `'. $repository->getDbTableName() .'` WHERE `'. $repository->getDbTableName() .'`.`'. $count_by_field .'` = `'. $this->getDbTableName() .'`.`id`) AS `'. $field_name .'`',
            'as' => false,
            'type' => 'string'
        ];

        return $this;
    }

    public function getSelectFields() {
        return $this->sql_select_fields;
    }

    public function getTotalSelectedRowsWithoutLimit() {
        return $this->total_count_rows;
    }

{'fraction_non_alphanumeric': 0.13892060592220445, 'fraction_numerical': 0.00045672527974423386, 'mean_word_length': 1.822234156820623, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 259, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2129712', 'n_tokens_mistral': 7787, 'n_tokens_neox': 7507, 'n_words': 1579}