<commit_before><?php
App::uses('AppModel', 'Model');
class Warninglist extends AppModel
{
    public $useTable = 'warninglists';

    public $recursive = -1;

    public $actsAs = array(
            'Containable',
    );

    public $validate = array(
        'name' => array(
            'rule' => array('valueNotEmpty'),
        ),
        'description' => array(
            'rule' => array('valueNotEmpty'),
        ),
        'version' => array(
            'rule' => array('numeric'),
        ),
    );

    public $hasMany = array(
            'WarninglistEntry' => array(
                'dependent' => true
            ),
            'WarninglistType' => array(
                'dependent' => true
            )
    );

    private $__tlds = array(
        'TLDs as known by IANA'
    );

    public function beforeValidate($options = array())
    {
        parent::beforeValidate();
        return true;
    }

    public function getTldLists()
    {
        return $this->__tlds;
    }

    public function update()
    {
        $directories = glob(APP . 'files' . DS . 'warninglists' . DS . 'lists' . DS . '*', GLOB_ONLYDIR);
        $updated = array();
        foreach ($directories as $dir) {
            $file = new File($dir . DS . 'list.json');
            $list = json_decode($file->read(), true);
            $file->close();
            if (!isset($list['version'])) {
                $list['version'] = 1;
            }
            if (!isset($list['type'])) {
                $list['type'] = 'string';
            } elseif (is_array($list['type'])) {
                $list['type'] = $list['type'][0];
            }
            $current = $this->find('first', array(
                    'conditions' => array('name' => $list['name']),
                    'recursive' => -1,
                    'fields' => array('*')
            ));
            if (empty($current) || $list['version'] > $current['Warninglist']['version']) {
                $result = $this->__updateList($list, $current);
                if (is_numeric($result)) {
                    $updated['success'][$result] = array('name' => $list['name'], 'new' => $list['version']);
                    if (!empty($current)) {
                        $updated['success'][$result]['old'] = $current['Warninglist']['version'];
                    }
                } else {
                    $updated['fails'][] = array('name' => $list['name'], 'fail' => json_encode($result));
                }
            }
        }
        $this->regenerateWarninglistCaches();
        return $updated;
    }

    public function quickDelete($id)
    {
        $result = $this->WarninglistEntry->deleteAll(
            array('WarninglistEntry.warninglist_id' => $id)
        );
        if ($result) {
            $result = $this->WarninglistType->deleteAll(
                array('WarninglistType.warninglist_id' => $id)
            );
        }
        if ($result) {
            $result = $this->delete($id, false);
        }
        return $result;
    }

    private function __updateList($list, $current)
    {
        $list['enabled'] = 0;
        $warninglist = array();
        if (!empty($current)) {
            if ($current['Warninglist']['enabled']) {
                $list['enabled'] = 1;
            }
            $this->quickDelete($current['Warninglist']['id']);
        }
        $fieldsToSave = array('name', 'version', 'description', 'type', 'enabled');
        foreach ($fieldsToSave as $fieldToSave) {
            $warninglist['Warninglist'][$fieldToSave] = $list[$fieldToSave];
        }
        $this->create();
        if ($this->save($warninglist)) {
            $db = $this->getDataSource();
            $values = array();
            foreach ($list['list'] as $value) {
                if (!empty($value)) {
                    $values[] = array('value' => $value, 'warninglist_id' => $this->id);
                }
            }
            unset($list['list']);
            $count = count($values);
            $values = array_chunk($values, 100);
            foreach ($values as $chunk) {
                $result = $db->insertMulti('warninglist_entries', array('value', 'warninglist_id'), $chunk);
            }
            if ($result) {
                $this->saveField('warninglist_entry_count', $count);
            } else {
                return 'Could not insert values.';
            }
            if (!empty($list['matching_attributes'])) {
                $values = array();
                foreach ($list['matching_attributes'] as $type) {
                    $values[] = array('type' => $type, 'warninglist_id' => $this->id);
                }
                $this->WarninglistType->saveMany($values);
            } else {
                $this->WarninglistType->create();
                $this->WarninglistType->save(array('WarninglistType' => array('type' => 'ALL', 'warninglist_id' => $this->id)));
            }
            return $this->id;
        } else {
            return $this->validationErrors;
        }
    }

    // regenerate the warninglist caches, but if an ID is passed along, only regen the entries for the given ID.
    // This allows us to enable/disable a single warninglist without regenerating all caches
    public function regenerateWarninglistCaches($id = false)
    {
        $redis = $this->setupRedis();
        if ($redis === false) {
            return false;
        }
        $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => array('enabled' => 1)));
        $this->cacheWarninglists($warninglists);
        foreach ($warninglists as $warninglist) {
            if ($id && $warninglist['Warninglist']['id'] != $id) {
                continue;
            }
            $entries = $this->WarninglistEntry->find('list', array(
                    'recursive' => -1,
                    'conditions' => array('warninglist_id' => $warninglist['Warninglist']['id']),
                    'fields' => array('value')
            ));
            $this->cacheWarninglistEntries($entries, $warninglist['Warninglist']['id']);
        }
        return true;
    }

    public function cacheWarninglists($warninglists)
    {
        $redis = $this->setupRedis();
        if ($redis !== false) {
            $redis->del('misp:warninglist_cache');
            foreach ($warninglists as $warninglist) {
                $redis->sAdd('misp:warninglist_cache', json_encode($warninglist));
            }
            return true;
        }
        return false;
    }

    public function cacheWarninglistEntries($warninglistEntries, $id)
    {
        $redis = $this->setupRedis();
        if ($redis !== false) {
            $key = 'misp:warninglist_entries_cache:' . $id;
            $redis->del($key);
            $redis->sAddArray($key, $warninglistEntries);
            return true;
        }
        return false;
    }

    public function getWarninglists($conditions)
    {
        $redis = $this->setupRedis();
        if ($redis !== false) {
            if ($redis->sCard('misp:warninglist_cache') === 0) {
                if (!empty($conditions)) {
                    $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => $conditions));
                } else {<commit_msg>fix: [internal] warninglist missing Redis::sAddArray() fixed<commit_after><?php
App::uses('AppModel', 'Model');
class Warninglist extends AppModel
{
    public $useTable = 'warninglists';

    public $recursive = -1;

    public $actsAs = array(
            'Containable',
    );

    public $validate = array(
        'name' => array(
            'rule' => array('valueNotEmpty'),
        ),
        'description' => array(
            'rule' => array('valueNotEmpty'),
        ),
        'version' => array(
            'rule' => array('numeric'),
        ),
    );

    public $hasMany = array(
            'WarninglistEntry' => array(
                'dependent' => true
            ),
            'WarninglistType' => array(
                'dependent' => true
            )
    );

    private $__tlds = array(
        'TLDs as known by IANA'
    );

    public function beforeValidate($options = array())
    {
        parent::beforeValidate();
        return true;
    }

    public function getTldLists()
    {
        return $this->__tlds;
    }

    public function update()
    {
        $directories = glob(APP . 'files' . DS . 'warninglists' . DS . 'lists' . DS . '*', GLOB_ONLYDIR);
        $updated = array();
        foreach ($directories as $dir) {
            $file = new File($dir . DS . 'list.json');
            $list = json_decode($file->read(), true);
            $file->close();
            if (!isset($list['version'])) {
                $list['version'] = 1;
            }
            if (!isset($list['type'])) {
                $list['type'] = 'string';
            } elseif (is_array($list['type'])) {
                $list['type'] = $list['type'][0];
            }
            $current = $this->find('first', array(
                    'conditions' => array('name' => $list['name']),
                    'recursive' => -1,
                    'fields' => array('*')
            ));
            if (empty($current) || $list['version'] > $current['Warninglist']['version']) {
                $result = $this->__updateList($list, $current);
                if (is_numeric($result)) {
                    $updated['success'][$result] = array('name' => $list['name'], 'new' => $list['version']);
                    if (!empty($current)) {
                        $updated['success'][$result]['old'] = $current['Warninglist']['version'];
                    }
                } else {
                    $updated['fails'][] = array('name' => $list['name'], 'fail' => json_encode($result));
                }
            }
        }
        $this->regenerateWarninglistCaches();
        return $updated;
    }

    public function quickDelete($id)
    {
        $result = $this->WarninglistEntry->deleteAll(
            array('WarninglistEntry.warninglist_id' => $id)
        );
        if ($result) {
            $result = $this->WarninglistType->deleteAll(
                array('WarninglistType.warninglist_id' => $id)
            );
        }
        if ($result) {
            $result = $this->delete($id, false);
        }
        return $result;
    }

    private function __updateList($list, $current)
    {
        $list['enabled'] = 0;
        $warninglist = array();
        if (!empty($current)) {
            if ($current['Warninglist']['enabled']) {
                $list['enabled'] = 1;
            }
            $this->quickDelete($current['Warninglist']['id']);
        }
        $fieldsToSave = array('name', 'version', 'description', 'type', 'enabled');
        foreach ($fieldsToSave as $fieldToSave) {
            $warninglist['Warninglist'][$fieldToSave] = $list[$fieldToSave];
        }
        $this->create();
        if ($this->save($warninglist)) {
            $db = $this->getDataSource();
            $values = array();
            foreach ($list['list'] as $value) {
                if (!empty($value)) {
                    $values[] = array('value' => $value, 'warninglist_id' => $this->id);
                }
            }
            unset($list['list']);
            $count = count($values);
            $values = array_chunk($values, 100);
            foreach ($values as $chunk) {
                $result = $db->insertMulti('warninglist_entries', array('value', 'warninglist_id'), $chunk);
            }
            if ($result) {
                $this->saveField('warninglist_entry_count', $count);
            } else {
                return 'Could not insert values.';
            }
            if (!empty($list['matching_attributes'])) {
                $values = array();
                foreach ($list['matching_attributes'] as $type) {
                    $values[] = array('type' => $type, 'warninglist_id' => $this->id);
                }
                $this->WarninglistType->saveMany($values);
            } else {
                $this->WarninglistType->create();
                $this->WarninglistType->save(array('WarninglistType' => array('type' => 'ALL', 'warninglist_id' => $this->id)));
            }
            return $this->id;
        } else {
            return $this->validationErrors;
        }
    }

    // regenerate the warninglist caches, but if an ID is passed along, only regen the entries for the given ID.
    // This allows us to enable/disable a single warninglist without regenerating all caches
    public function regenerateWarninglistCaches($id = false)
    {
        $redis = $this->setupRedis();
        if ($redis === false) {
            return false;
        }
        $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => array('enabled' => 1)));
        $this->cacheWarninglists($warninglists);
        foreach ($warninglists as $warninglist) {
            if ($id && $warninglist['Warninglist']['id'] != $id) {
                continue;
            }
            $entries = $this->WarninglistEntry->find('list', array(
                    'recursive' => -1,
                    'conditions' => array('warninglist_id' => $warninglist['Warninglist']['id']),
                    'fields' => array('value')
            ));
            $this->cacheWarninglistEntries($entries, $warninglist['Warninglist']['id']);
        }
        return true;
    }

    public function cacheWarninglists($warninglists)
    {
        $redis = $this->setupRedis();
        if ($redis !== false) {
            $redis->del('misp:warninglist_cache');
            foreach ($warninglists as $warninglist) {
                $redis->sAdd('misp:warninglist_cache', json_encode($warninglist));
            }
            return true;
        }
        return false;
    }

    public function cacheWarninglistEntries($warninglistEntries, $id)
    {
        $redis = $this->setupRedis();
        if ($redis !== false) {
            $key = 'misp:warninglist_entries_cache:' . $id;
            $redis->del($key);
            foreach ($warninglistEntries as $entry) {
                $redis->sAdd('misp:warninglist_entries_cache:' . $id, $entry);
            }
            return true;
        }
        return false;
    }

    public function getWarninglists($conditions)
    {
        $redis = $this->setupRedis();
        if ($redis !== false) {
            if ($redis->sCard('misp:warninglist_cache') === 0) {
                if (!empty($conditions)) {
                    $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => $conditions));
                } else {
{'fraction_non_alphanumeric': 0.16347589952477937, 'fraction_numerical': 0.001629327902240326, 'mean_word_length': 1.7580977345066466, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 163, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2727152', 'n_tokens_mistral': 4057, 'n_tokens_neox': 3929, 'n_words': 782}