<commit_before><?php
App::uses('AppModel', 'Model');
App::uses('TmpFileTool', 'Tools');

/**
 * @property Event $Event
 * @property SharingGroup $SharingGroup
 */
class MispObject extends AppModel
{
    public $name = 'Object';
    public $alias = 'Object';

    public $useTable = 'objects';

    public $actsAs = array(
            'Containable',
            'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
                'userModel' => 'User',
                'userKey' => 'user_id',
                'change' => 'full'),
    );

    public $belongsTo = array(
        'Event' => array(
            'className' => 'Event',
            'foreignKey' => 'event_id'
        ),
        'SharingGroup' => array(
            'className' => 'SharingGroup',
            'foreignKey' => 'sharing_group_id'
        ),
        'ObjectTemplate' => array(
            'className' => 'ObjectTemplate',
            'foreignKey' => false,
            'dependent' => false,
            'conditions' => array('MispObject.template_uuid' => 'ObjectTemplate.uuid')
        )
    );

    public $hasMany = array(
        'Attribute' => array(
            'className' => 'Attribute',
            'dependent' => true,
        ),
        'ObjectReference' => array(
            'className' => 'ObjectReference',
            'dependent' => true,
            'foreignKey' => 'object_id'
        ),
    );

    public $validFormats = array(
        'json' => array('json', 'JsonExport', 'json')
    );

    public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' Sharing Group', 5 => 'Inherit');

    public $validate = array(
        'uuid' => array(
            'uuid' => array(
                'rule' => 'uuid',
                'message' => 'Please provide a valid RFC 4122 UUID'
            ),
            'unique' => array(
                'rule' => 'isUnique',
                'message' => 'The UUID provided is not unique',
                'required' => 'create'
            ),
        ),
        'first_seen' => array(
            'rule' => array('datetimeOrNull'),
            'required' => false,
            'message' => array('Invalid ISO 8601 format')
        ),
        'last_seen' => array(
            'rule' => array('datetimeOrNull'),
            'required' => false,
            'message' => array('Invalid ISO 8601 format')
        )
    );

    public function buildFilterConditions(&$params)
    {
        $conditions = [];
        if (isset($params['wildcard'])) {
            $temp = array();
            $options = array(
                'filter' => 'wildcard',
                'scope' => 'Object',
                'pop' => false,
                'context' => 'Event'
            );
            $conditions['AND'][] = array('OR' => $this->Event->set_filter_wildcard_attributes($params, $temp, $options));
        } else {
            if (isset($params['ignore'])) {
                $params['to_ids'] = array(0, 1);
                $params['published'] = array(0, 1);
            }
            $simple_params = array(
                'Object' => array(
                    'object_name' => array('function' => 'set_filter_object_name'),
                    'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'),
                    'object_template_version' => array('function' => 'set_filter_object_template_version'),
                    'deleted' => array('function' => 'set_filter_deleted')
                ),
                'Event' => array(
                    'eventid' => array('function' => 'set_filter_eventid'),
                    'eventinfo' => array('function' => 'set_filter_eventinfo'),
                    'ignore' => array('function' => 'set_filter_ignore'),
                    'from' => array('function' => 'set_filter_timestamp'),
                    'to' => array('function' => 'set_filter_timestamp'),
                    'date' => array('function' => 'set_filter_date'),
                    'tags' => array('function' => 'set_filter_tags'),
                    'last' => array('function' => 'set_filter_timestamp', 'pop' => true),
                    'timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
                    'event_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
                    'publish_timestamp' => array('function' => 'set_filter_timestamp'),
                    'org' => array('function' => 'set_filter_org'),
                    'uuid' => array('function' => 'set_filter_uuid'),
                    'published' => array('function' => 'set_filter_published')
                ),
                'Attribute' => array(
                    'value' => array('function' => 'set_filter_value'),
                    'category' => array('function' => 'set_filter_simple_attribute'),
                    'type' => array('function' => 'set_filter_simple_attribute'),
                    'object_relation' => array('function' => 'set_filter_simple_attribute'),
                    'tags' => array('function' => 'set_filter_tags', 'pop' => true),
                    'uuid' => array('function' => 'set_filter_uuid'),
                    'deleted' => array('function' => 'set_filter_deleted'),
                    'timestamp' => array('function' => 'set_filter_timestamp'),
                    'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
                    'first_seen' => array('function' => 'set_filter_seen'),
                    'last_seen' => array('function' => 'set_filter_seen'),
                    'to_ids' => array('function' => 'set_filter_to_ids'),
                    'comment' => array('function' => 'set_filter_comment')
                )
            );
            foreach ($params as $param => $paramData) {
                foreach ($simple_params as $scope => $simple_param_scoped) {
                    if (isset($simple_param_scoped[$param]) && isset($params[$param]) && $params[$param] !== false) {
                        $options = array(
                            'filter' => $param,
                            'scope' => $scope,
                            'pop' => !empty($simple_param_scoped[$param]['pop']),
                            'context' => 'Attribute'
                        );
                        if ($scope === 'Attribute') {
                            $subQueryOptions = array(
                                'fields' => ['Attribute.object_id'],
                                'group' => 'Attribute.object_id',
                                'recursive' => -1,
                                'conditions' => array(
                                    'Attribute.object_id NOT' => 0,
                                    $this->Event->{$simple_param_scoped[$param]['function']}($params, $conditions, $options)
                                )
                            );
                            $conditions['AND'][] = $this->subQueryGenerator($this->Attribute, $subQueryOptions, 'Object.id');
                        } else {
                            $conditions = $this->Event->{$simple_param_scoped[$param]['function']}($params, $conditions, $options);
                        }
                    }
                }
            }
        }
        return $conditions;
    }

     // check whether the variable is null or datetime
     public function datetimeOrNull($fields)
     {
         $k = array_keys($fields)[0];
         $seen = $fields[$k];
         try {
             new DateTime($seen);
             $returnValue = true;
         } catch (Exception $e) {
             $returnValue = false;
         }
         return $returnValue || is_null($seen);
     }

    public function afterFind($results, $primary = false)
    {
        foreach ($results as $k => $v) {
            $results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias);
        }
        return $results;
    }

    public function beforeSave($options = array()) {
        $this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
    }

    public function beforeValidate($options = array())
    {
        parent::beforeValidate();
        if (empty($this->data[$this->alias]['comment'])) {
            $this->data[$this->alias]['comment'] = "";
        }
        // generate UUID if it doesn't exist
        if (empty($this->data[$this->alias]['uuid'])) {
            $this->data[$this->alias]['uuid'] = CakeText::uuid();
        }
        // generate timestamp if it doesn't exist
        if (empty($this->data[$this->alias]['timestamp'])) {
            $date = new DateTime();
            $this->data[$this->alias]['timestamp'] = $date->getTimestamp();
        }
        // parse first_seen different formats
        if (isset($this->data[$this->alias]['first_seen'])) {
            $this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen'];
        }
        // parse last_seen different formats
        if (isset($this->data[$this->alias]['last_seen'])) {
            $this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen'];
        }
        if (empty($this->data[$this->alias]['template_version'])) {
            $this->data[$this->alias]['template_version'] = 1;
        }
        if (isset($this->data[$this->alias]['deleted']) && empty($this->data[$this->alias]['deleted'])) {
            $this->data[$this->alias]['deleted'] = 0;
        }
        if (!isset($this->data[$this->alias]['distribution']) || $this->data['Object']['distribution'] != 4) {
            $this->data['Object']['sharing_group_id'] = 0;
        }
        if (!isset($this->data[$this->alias]['distribution'])) {
            $this->data['Object']['distribution'] = 5;
        }
        return true;
    }

    public function afterSave($created, $options = array())
    {
        $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') &&
            Configure::read('Plugin.ZeroMQ_object_notifications_enable') &&
            empty($this->data['Object']['skip_zmq']);
        $kafkaTopic = Configure::read('Plugin.Kafka_object_notifications_topic');
        $pubToKafka = Configure::read('Plugin.Kafka_enable') &&
            Configure::read('Plugin.Kafka_object_notifications_enable') &&
            !empty($kafkaTopic) &&
            empty($this->data['Object']['skip_kafka']);
        if ($pubToZmq || $pubToKafka) {
            $object = $this->find('first', array(
                'conditions' => array('Object.id' => $this->id),
                'recursive' => -1
            ));
            $action = $created ? 'add' : 'edit';
            if (!empty($this->data['Object']['deleted'])) {
                $action = 'soft-delete';
            }
            if ($pubToZmq) {
                $pubSubTool = $this->getPubSubTool();
                $pubSubTool->object_save($object, $action);
            }
            if ($pubToKafka) {
                $kafkaPubTool = $this->getKafkaPubTool();
                $kafkaPubTool->publishJson($kafkaTopic, $object, $action);
            }
        }
        return true;
    }

    public function beforeDelete($cascade = true)
    {
        if (!empty($this->data['Object']['id'])) {
            $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_notifications_enable');
            $kafkaTopic = Configure::read('Plugin.Kafka_object_notifications_topic');
            $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_object_notifications_enable') && !empty($kafkaTopic);
            if ($pubToZmq || $pubToKafka) {
                $object = $this->find('first', array(
                    'recursive' => -1,
                    'conditions' => array('Object.id' => $this->data['Object']['id'])
                ));
                if ($pubToZmq) {
                    $pubSubTool = $this->getPubSubTool();
                    $pubSubTool->object_save($object, 'delete');
                }
                if ($pubToKafka) {
                    $kafkaPubTool = $this->getKafkaPubTool();
                    $kafkaPubTool->publishJson($kafkaTopic, $object, 'delete');
                }
            }
        }
    }

    public function afterDelete()
    {
        if (!empty($this->data[$this->alias]['id'])) {
            $this->ObjectReference->deleteAll(
                array(
                    'ObjectReference.referenced_type' => 1,
                    'ObjectReference.referenced_id' => $this->data[$this->alias]['id'],
                ),
                false
            );
        }
    }

    public function checkForDuplicateObjects($object, $eventId)
    {
        $newObjectAttributes = array();
        $existingObjectAttributes = array();
        foreach ($object['Attribute'] as $attribute) {
            $newObjectAttributes[] = hash(
                'sha256',
                $attribute['object_relation'] . $attribute['category'] . $attribute['type'] . $attribute['value']
            );
        }
        $newObjectAttributeCount = count($newObjectAttributes);
        $existingObjects = $this->find('all', array(
            'recursive' => -1,
            'contain' => array(
                'Attribute' => array(
                    'fields' => array('value', 'type', 'category', 'object_relation'),
                    'conditions' => array('Attribute.deleted' => 0)
                )
            ),
            'fields' => array('template_uuid'),
            'conditions' => array('template_uuid' => $object['Object']['template_uuid'], 'Object.deleted' => 0)
        ));
        $oldObjects = array();
        foreach ($existingObjects as $k => $existingObject) {
            $temp = array();
            if (!empty($existingObject['Attribute']) && $newObjectAttributeCount == count($existingObject['Attribute'])) {
                foreach ($existingObject['Attribute'] as $existingAttribute) {
                    $temp[] = hash(
                        'sha256',
                        $attribute['object_relation'] . $existingAttribute['category'] . $existingAttribute['type'] . $existingAttribute['value']
                    );
                }
                if (empty(array_diff($temp, $newObjectAttributes))) {
                    return true;
                }
            }
        }
        return false;
    }

    public function saveObject($object, $eventId, $template = false, $user, $errorBehaviour = 'drop', $breakOnDuplicate = false)
    {
        if ($breakOnDuplicate) {
            $duplicate = $this->checkForDuplicateObjects($object, $eventId);
            if ($duplicate) {
                return array('value' => array('Duplicate object found. Since breakOnDuplicate is set the object will not be added.'));
            }
        }
        $this->create();
        $templateFields = array(
            'name' => 'name',
            'meta-category' => 'meta-category',
            'description' => 'description',
            'template_version' => 'version',
            'template_uuid' => 'uuid'
        );
        if ($template) {
            foreach ($templateFields as $k => $v) {
                $object['Object'][$k] = $template['ObjectTemplate'][$v];
            }
        } else {
            foreach ($templateFields as $k => $v) {
                if (!isset($object['Object'][$k])) {
                    return 'No valid template found and object lacking template information. (' . $k . ')';
                }
            }
        }
        $object['Object']['event_id'] = $eventId;
        $result = false;
        if ($this->save($object)) {
            $result = $this->id;
            foreach ($object['Attribute'] as $k => $attribute) {
                $object['Attribute'][$k]['object_id'] = $this->id;
                if (
                    (!array_key_exists('first_seen', $object['Attribute'][$k]) || is_null($object['Attribute'][$k]['first_seen'])) &&
                    (array_key_exists('first_seen', $object['Object']) && !is_null($object['Object']['first_seen']))
                ) {
                    $object['Attribute'][$k]['first_seen'] = $object['Object']['first_seen'];
                }
                if (
                    (!array_key_exists('last_seen', $object['Attribute'][$k]) || is_null($object['Attribute'][$k]['last_seen'])) &&
                    (array_key_exists('last_seen', $object['Object']) &&  !is_null($object['Object']['last_seen']))
                ) {
                    $object['Attribute'][$k]['last_seen'] = $object['Object']['last_seen'];
                }
            }
            $this->Attribute->saveAttributes($object['Attribute'], $user);
        } else {
            $result = $this->validationErrors;
        }
        return $result;
    }

    public function buildConditions(array $user)
    {
        $conditions = array();
        if (!$user['Role']['perm_site_admin']) {
            $sgids = $this->Event->cacheSgids($user, true);
            $conditions = array(
                'AND' => array(
                    'OR' => array(
                        array(
                            'AND' => array(
                                'Event.org_id' => $user['org_id'],
                            )
                        ),
                        array(
                            'AND' => array(
                                $this->Event->createEventConditions($user),
                                'OR' => array(
                                    'Object.distribution' => array(1, 2, 3, 5),
                                    'AND' => array(
                                        'Object.distribution' => 4,
                                        'Object.sharing_group_id' => $sgids,
                                    )
                                )
                            )
                        )
                    )
                )
            );
        }
        return $conditions;
    }

    public function fetchObjectSimple($user, $options = array())
    {
        $params = array(
            'conditions' => $this->buildConditions($user),
            'fields' => array(),
            'recursive' => -1
        );
        if (isset($options['conditions'])) {
            $params['conditions']['AND'][] = $options['conditions'];
        }
        if (isset($options['fields'])) {
            $params['fields'] = $options['fields'];
        }
        $results = $this->find('all', array(
            'conditions' => $params['conditions'],
            'recursive' => -1,
            'fields' => $params['fields'],
            'contain' => array('Event' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id')),
            'sort' => false
        ));
        return $results;
    }

    // Method that fetches all objects
    // very flexible, it's basically a replacement for find, with the addition that it restricts access based on user
    // options:
    //     fields
    //     contain
    //     conditions
    //     order
    //     group
    public function fetchObjects($user, $options = array())
    {
        $attributeConditions = array();
        if (!$user['Role']['perm_site_admin']) {
            $sgids = $this->Event->cacheSgids($user, true);
            $attributeConditions = array(
                'OR' => array(
                    array(
                        '(SELECT events.org_id FROM events WHERE events.id = Attribute.event_id)' => $user['org_id']
                    ),
                    array(
                        'OR' => array(
                            'Attribute.distribution' => array(1, 2, 3, 5),
                            array(
                                'Attribute.distribution' => 4,
                                'Attribute.sharing_group_id' => $sgids,
                            )
                        )
                    )
                )
            );
        }
        $params = array(
            'conditions' => $this->buildConditions($user),
            'recursive' => -1,
            'contain' => array(
                'Event' => array(
                    'fields' => array('id', 'info', 'org_id', 'orgc_id'),
                ),
                'Attribute' => array(
                    'conditions' => $attributeConditions,
                    //'ShadowAttribute',
                    'AttributeTag' => array(
                        'Tag'
                    )
                )
            ),
        );
        if (!empty($options['metadata'])) {
            unset($params['contain']['Attribute']);
        }
        if (empty($options['metadata']) && empty($options['includeAllTags'])) {
            $params['contain']['Attribute']['AttributeTag']['Tag']['conditions']['exportable'] = 1;
        }
        if (isset($options['contain'])) {
            $params['contain'] = array_merge_recursive($params['contain'], $options['contain']);
        } else {
            $option['contain']['Event']['fields'] = array('id', 'info', 'org_id', 'orgc_id');
        }
        if (
            empty($options['metadata']) &&
            Configure::read('MISP.proposals_block_attributes') &&
            isset($options['conditions']['AND']['Attribute.to_ids']) &&
            $options['conditions']['AND']['Attribute.to_ids'] == 1
        ) {
            $this->Attribute->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id'))));
            $proposalRestriction =  array(
                    'ShadowAttribute' => array(
                            'conditions' => array(
                                    'AND' => array(
                                            'ShadowAttribute.deleted' => 0,
                                            'OR' => array(
                                                    'ShadowAttribute.proposal_to_delete' => 1,
                                                    'ShadowAttribute.to_ids' => 0
                                            )
                                    )
                            ),
                            'fields' => array('ShadowAttribute.id')
                    )
            );
            $params['contain'] = array_merge($params['contain']['Attribute'], $proposalRestriction);
        }
        if (isset($options['conditions'])) {
            $params['conditions']['AND'][] = $options['conditions'];
        }
        if (isset($options['order'])) {
            $params['order'] = $options['order'];
        }
        if (!isset($options['withAttachments'])) {
            $options['withAttachments'] = false;
        } else ($params['order'] = array());
        if (!isset($options['enforceWarninglist'])) {
            $options['enforceWarninglist'] = false;
        }
        if (empty($options['metadata']) && (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted'])) {
            $params['contain']['Attribute']['conditions']['AND']['Attribute.deleted'] = 0;
        }
        if (isset($options['group'])) {
            $params['group'] = array_merge(array('Object.id'), $options['group']);
        }
        if (isset($options['limit'])) {
            $params['limit'] = $options['limit'];
            if (isset($options['page'])) {
                $params['page'] = $options['page'];
            }
        }
        $results = $this->find('all', $params);
        if ($options['enforceWarninglist']) {
            $this->Warninglist = ClassRegistry::init('Warninglist');
            $warninglists = $this->Warninglist->fetchForEventView();
        }
        $results = array_values($results);
        $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
        if (empty($options['metadata'])) {
            foreach ($results as $key => $objects) {
                foreach ($objects as $key2 => $attribute) {
                    if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'], $this->Warninglist)) {
                        unset($results[$key][$key2]);
                        continue;
                    }
                    if ($proposals_block_attributes) {
                        if (!empty($attribute['ShadowAttribute'])) {
                            unset($results[$key][$key2]);
                        } else {
                            unset($results[$key][$key2]['ShadowAttribute']);
                        }
                    }
                    if ($options['withAttachments']) {
                        if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
                            $encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
                            $results[$key][$key2]['Attribute']['data'] = $encodedFile;
                        }
                    }
                }
            }
        }
        return $results;
    }

    /*
     * Prepare the template form view's data, setting defaults, sorting elements
     */
    public function prepareTemplate($template, $request = array())
    {
        $temp = array();
        usort($template['ObjectTemplateElement'], function ($a, $b) {
            return $a['ui-priority'] < $b['ui-priority'];
        });
        $request_rearranged = array();
        $template_object_elements = $template['ObjectTemplateElement'];
        unset($template['ObjectTemplateElement']);
        if (!empty($request['Attribute'])) {
            foreach ($request['Attribute'] as $attribute) {
                $request_rearranged[$attribute['object_relation']][] = $attribute;
            }<commit_msg>fix: [internal] Fetching objects with attachments<commit_after><?php
App::uses('AppModel', 'Model');
App::uses('TmpFileTool', 'Tools');

/**
 * @property Event $Event
 * @property SharingGroup $SharingGroup
 * @property Attribute $Attribute
 */
class MispObject extends AppModel
{
    public $name = 'Object';
    public $alias = 'Object';

    public $useTable = 'objects';

    public $actsAs = array(
            'Containable',
            'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
                'userModel' => 'User',
                'userKey' => 'user_id',
                'change' => 'full'),
    );

    public $belongsTo = array(
        'Event' => array(
            'className' => 'Event',
            'foreignKey' => 'event_id'
        ),
        'SharingGroup' => array(
            'className' => 'SharingGroup',
            'foreignKey' => 'sharing_group_id'
        ),
        'ObjectTemplate' => array(
            'className' => 'ObjectTemplate',
            'foreignKey' => false,
            'dependent' => false,
            'conditions' => array('MispObject.template_uuid' => 'ObjectTemplate.uuid')
        )
    );

    public $hasMany = array(
        'Attribute' => array(
            'className' => 'Attribute',
            'dependent' => true,
        ),
        'ObjectReference' => array(
            'className' => 'ObjectReference',
            'dependent' => true,
            'foreignKey' => 'object_id'
        ),
    );

    public $validFormats = array(
        'json' => array('json', 'JsonExport', 'json')
    );

    public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' Sharing Group', 5 => 'Inherit');

    public $validate = array(
        'uuid' => array(
            'uuid' => array(
                'rule' => 'uuid',
                'message' => 'Please provide a valid RFC 4122 UUID'
            ),
            'unique' => array(
                'rule' => 'isUnique',
                'message' => 'The UUID provided is not unique',
                'required' => 'create'
            ),
        ),
        'first_seen' => array(
            'rule' => array('datetimeOrNull'),
            'required' => false,
            'message' => array('Invalid ISO 8601 format')
        ),
        'last_seen' => array(
            'rule' => array('datetimeOrNull'),
            'required' => false,
            'message' => array('Invalid ISO 8601 format')
        )
    );

    public function buildFilterConditions(&$params)
    {
        $conditions = [];
        if (isset($params['wildcard'])) {
            $temp = array();
            $options = array(
                'filter' => 'wildcard',
                'scope' => 'Object',
                'pop' => false,
                'context' => 'Event'
            );
            $conditions['AND'][] = array('OR' => $this->Event->set_filter_wildcard_attributes($params, $temp, $options));
        } else {
            if (isset($params['ignore'])) {
                $params['to_ids'] = array(0, 1);
                $params['published'] = array(0, 1);
            }
            $simple_params = array(
                'Object' => array(
                    'object_name' => array('function' => 'set_filter_object_name'),
                    'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'),
                    'object_template_version' => array('function' => 'set_filter_object_template_version'),
                    'deleted' => array('function' => 'set_filter_deleted')
                ),
                'Event' => array(
                    'eventid' => array('function' => 'set_filter_eventid'),
                    'eventinfo' => array('function' => 'set_filter_eventinfo'),
                    'ignore' => array('function' => 'set_filter_ignore'),
                    'from' => array('function' => 'set_filter_timestamp'),
                    'to' => array('function' => 'set_filter_timestamp'),
                    'date' => array('function' => 'set_filter_date'),
                    'tags' => array('function' => 'set_filter_tags'),
                    'last' => array('function' => 'set_filter_timestamp', 'pop' => true),
                    'timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
                    'event_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
                    'publish_timestamp' => array('function' => 'set_filter_timestamp'),
                    'org' => array('function' => 'set_filter_org'),
                    'uuid' => array('function' => 'set_filter_uuid'),
                    'published' => array('function' => 'set_filter_published')
                ),
                'Attribute' => array(
                    'value' => array('function' => 'set_filter_value'),
                    'category' => array('function' => 'set_filter_simple_attribute'),
                    'type' => array('function' => 'set_filter_simple_attribute'),
                    'object_relation' => array('function' => 'set_filter_simple_attribute'),
                    'tags' => array('function' => 'set_filter_tags', 'pop' => true),
                    'uuid' => array('function' => 'set_filter_uuid'),
                    'deleted' => array('function' => 'set_filter_deleted'),
                    'timestamp' => array('function' => 'set_filter_timestamp'),
                    'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
                    'first_seen' => array('function' => 'set_filter_seen'),
                    'last_seen' => array('function' => 'set_filter_seen'),
                    'to_ids' => array('function' => 'set_filter_to_ids'),
                    'comment' => array('function' => 'set_filter_comment')
                )
            );
            foreach ($params as $param => $paramData) {
                foreach ($simple_params as $scope => $simple_param_scoped) {
                    if (isset($simple_param_scoped[$param]) && isset($params[$param]) && $params[$param] !== false) {
                        $options = array(
                            'filter' => $param,
                            'scope' => $scope,
                            'pop' => !empty($simple_param_scoped[$param]['pop']),
                            'context' => 'Attribute'
                        );
                        if ($scope === 'Attribute') {
                            $subQueryOptions = array(
                                'fields' => ['Attribute.object_id'],
                                'group' => 'Attribute.object_id',
                                'recursive' => -1,
                                'conditions' => array(
                                    'Attribute.object_id NOT' => 0,
                                    $this->Event->{$simple_param_scoped[$param]['function']}($params, $conditions, $options)
                                )
                            );
                            $conditions['AND'][] = $this->subQueryGenerator($this->Attribute, $subQueryOptions, 'Object.id');
                        } else {
                            $conditions = $this->Event->{$simple_param_scoped[$param]['function']}($params, $conditions, $options);
                        }
                    }
                }
            }
        }
        return $conditions;
    }

     // check whether the variable is null or datetime
     public function datetimeOrNull($fields)
     {
         $k = array_keys($fields)[0];
         $seen = $fields[$k];
         try {
             new DateTime($seen);
             $returnValue = true;
         } catch (Exception $e) {
             $returnValue = false;
         }
         return $returnValue || is_null($seen);
     }

    public function afterFind($results, $primary = false)
    {
        foreach ($results as $k => $v) {
            $results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias);
        }
        return $results;
    }

    public function beforeSave($options = array()) {
        $this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
    }

    public function beforeValidate($options = array())
    {
        parent::beforeValidate();
        if (empty($this->data[$this->alias]['comment'])) {
            $this->data[$this->alias]['comment'] = "";
        }
        // generate UUID if it doesn't exist
        if (empty($this->data[$this->alias]['uuid'])) {
            $this->data[$this->alias]['uuid'] = CakeText::uuid();
        }
        // generate timestamp if it doesn't exist
        if (empty($this->data[$this->alias]['timestamp'])) {
            $date = new DateTime();
            $this->data[$this->alias]['timestamp'] = $date->getTimestamp();
        }
        // parse first_seen different formats
        if (isset($this->data[$this->alias]['first_seen'])) {
            $this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen'];
        }
        // parse last_seen different formats
        if (isset($this->data[$this->alias]['last_seen'])) {
            $this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen'];
        }
        if (empty($this->data[$this->alias]['template_version'])) {
            $this->data[$this->alias]['template_version'] = 1;
        }
        if (isset($this->data[$this->alias]['deleted']) && empty($this->data[$this->alias]['deleted'])) {
            $this->data[$this->alias]['deleted'] = 0;
        }
        if (!isset($this->data[$this->alias]['distribution']) || $this->data['Object']['distribution'] != 4) {
            $this->data['Object']['sharing_group_id'] = 0;
        }
        if (!isset($this->data[$this->alias]['distribution'])) {
            $this->data['Object']['distribution'] = 5;
        }
        return true;
    }

    public function afterSave($created, $options = array())
    {
        $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') &&
            Configure::read('Plugin.ZeroMQ_object_notifications_enable') &&
            empty($this->data['Object']['skip_zmq']);
        $kafkaTopic = Configure::read('Plugin.Kafka_object_notifications_topic');
        $pubToKafka = Configure::read('Plugin.Kafka_enable') &&
            Configure::read('Plugin.Kafka_object_notifications_enable') &&
            !empty($kafkaTopic) &&
            empty($this->data['Object']['skip_kafka']);
        if ($pubToZmq || $pubToKafka) {
            $object = $this->find('first', array(
                'conditions' => array('Object.id' => $this->id),
                'recursive' => -1
            ));
            $action = $created ? 'add' : 'edit';
            if (!empty($this->data['Object']['deleted'])) {
                $action = 'soft-delete';
            }
            if ($pubToZmq) {
                $pubSubTool = $this->getPubSubTool();
                $pubSubTool->object_save($object, $action);
            }
            if ($pubToKafka) {
                $kafkaPubTool = $this->getKafkaPubTool();
                $kafkaPubTool->publishJson($kafkaTopic, $object, $action);
            }
        }
        return true;
    }

    public function beforeDelete($cascade = true)
    {
        if (!empty($this->data['Object']['id'])) {
            $pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_notifications_enable');
            $kafkaTopic = Configure::read('Plugin.Kafka_object_notifications_topic');
            $pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_object_notifications_enable') && !empty($kafkaTopic);
            if ($pubToZmq || $pubToKafka) {
                $object = $this->find('first', array(
                    'recursive' => -1,
                    'conditions' => array('Object.id' => $this->data['Object']['id'])
                ));
                if ($pubToZmq) {
                    $pubSubTool = $this->getPubSubTool();
                    $pubSubTool->object_save($object, 'delete');
                }
                if ($pubToKafka) {
                    $kafkaPubTool = $this->getKafkaPubTool();
                    $kafkaPubTool->publishJson($kafkaTopic, $object, 'delete');
                }
            }
        }
    }

    public function afterDelete()
    {
        if (!empty($this->data[$this->alias]['id'])) {
            $this->ObjectReference->deleteAll(
                array(
                    'ObjectReference.referenced_type' => 1,
                    'ObjectReference.referenced_id' => $this->data[$this->alias]['id'],
                ),
                false
            );
        }
    }

    public function checkForDuplicateObjects($object, $eventId)
    {
        $newObjectAttributes = array();
        $existingObjectAttributes = array();
        foreach ($object['Attribute'] as $attribute) {
            $newObjectAttributes[] = hash(
                'sha256',
                $attribute['object_relation'] . $attribute['category'] . $attribute['type'] . $attribute['value']
            );
        }
        $newObjectAttributeCount = count($newObjectAttributes);
        $existingObjects = $this->find('all', array(
            'recursive' => -1,
            'contain' => array(
                'Attribute' => array(
                    'fields' => array('value', 'type', 'category', 'object_relation'),
                    'conditions' => array('Attribute.deleted' => 0)
                )
            ),
            'fields' => array('template_uuid'),
            'conditions' => array('template_uuid' => $object['Object']['template_uuid'], 'Object.deleted' => 0)
        ));
        $oldObjects = array();
        foreach ($existingObjects as $k => $existingObject) {
            $temp = array();
            if (!empty($existingObject['Attribute']) && $newObjectAttributeCount == count($existingObject['Attribute'])) {
                foreach ($existingObject['Attribute'] as $existingAttribute) {
                    $temp[] = hash(
                        'sha256',
                        $attribute['object_relation'] . $existingAttribute['category'] . $existingAttribute['type'] . $existingAttribute['value']
                    );
                }
                if (empty(array_diff($temp, $newObjectAttributes))) {
                    return true;
                }
            }
        }
        return false;
    }

    public function saveObject($object, $eventId, $template = false, $user, $errorBehaviour = 'drop', $breakOnDuplicate = false)
    {
        if ($breakOnDuplicate) {
            $duplicate = $this->checkForDuplicateObjects($object, $eventId);
            if ($duplicate) {
                return array('value' => array('Duplicate object found. Since breakOnDuplicate is set the object will not be added.'));
            }
        }
        $this->create();
        $templateFields = array(
            'name' => 'name',
            'meta-category' => 'meta-category',
            'description' => 'description',
            'template_version' => 'version',
            'template_uuid' => 'uuid'
        );
        if ($template) {
            foreach ($templateFields as $k => $v) {
                $object['Object'][$k] = $template['ObjectTemplate'][$v];
            }
        } else {
            foreach ($templateFields as $k => $v) {
                if (!isset($object['Object'][$k])) {
                    return 'No valid template found and object lacking template information. (' . $k . ')';
                }
            }
        }
        $object['Object']['event_id'] = $eventId;
        $result = false;
        if ($this->save($object)) {
            $result = $this->id;
            foreach ($object['Attribute'] as $k => $attribute) {
                $object['Attribute'][$k]['object_id'] = $this->id;
                if (
                    (!array_key_exists('first_seen', $object['Attribute'][$k]) || is_null($object['Attribute'][$k]['first_seen'])) &&
                    (array_key_exists('first_seen', $object['Object']) && !is_null($object['Object']['first_seen']))
                ) {
                    $object['Attribute'][$k]['first_seen'] = $object['Object']['first_seen'];
                }
                if (
                    (!array_key_exists('last_seen', $object['Attribute'][$k]) || is_null($object['Attribute'][$k]['last_seen'])) &&
                    (array_key_exists('last_seen', $object['Object']) &&  !is_null($object['Object']['last_seen']))
                ) {
                    $object['Attribute'][$k]['last_seen'] = $object['Object']['last_seen'];
                }
            }
            $this->Attribute->saveAttributes($object['Attribute'], $user);
        } else {
            $result = $this->validationErrors;
        }
        return $result;
    }

    public function buildConditions(array $user)
    {
        $conditions = array();
        if (!$user['Role']['perm_site_admin']) {
            $sgids = $this->Event->cacheSgids($user, true);
            $conditions = array(
                'AND' => array(
                    'OR' => array(
                        array(
                            'AND' => array(
                                'Event.org_id' => $user['org_id'],
                            )
                        ),
                        array(
                            'AND' => array(
                                $this->Event->createEventConditions($user),
                                'OR' => array(
                                    'Object.distribution' => array(1, 2, 3, 5),
                                    'AND' => array(
                                        'Object.distribution' => 4,
                                        'Object.sharing_group_id' => $sgids,
                                    )
                                )
                            )
                        )
                    )
                )
            );
        }
        return $conditions;
    }

    public function fetchObjectSimple($user, $options = array())
    {
        $params = array(
            'conditions' => $this->buildConditions($user),
            'fields' => array(),
            'recursive' => -1
        );
        if (isset($options['conditions'])) {
            $params['conditions']['AND'][] = $options['conditions'];
        }
        if (isset($options['fields'])) {
            $params['fields'] = $options['fields'];
        }
        $results = $this->find('all', array(
            'conditions' => $params['conditions'],
            'recursive' => -1,
            'fields' => $params['fields'],
            'contain' => array('Event' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id')),
            'sort' => false
        ));
        return $results;
    }

    // Method that fetches all objects
    // very flexible, it's basically a replacement for find, with the addition that it restricts access based on user
    // options:
    //     fields
    //     contain
    //     conditions
    //     order
    //     group
    public function fetchObjects($user, $options = array())
    {
        $attributeConditions = array();
        if (!$user['Role']['perm_site_admin']) {
            $sgids = $this->Event->cacheSgids($user, true);
            $attributeConditions = array(
                'OR' => array(
                    array(
                        '(SELECT events.org_id FROM events WHERE events.id = Attribute.event_id)' => $user['org_id']
                    ),
                    array(
                        'OR' => array(
                            'Attribute.distribution' => array(1, 2, 3, 5),
                            array(
                                'Attribute.distribution' => 4,
                                'Attribute.sharing_group_id' => $sgids,
                            )
                        )
                    )
                )
            );
        }
        $params = array(
            'conditions' => $this->buildConditions($user),
            'recursive' => -1,
            'contain' => array(
                'Event' => array(
                    'fields' => array('id', 'info', 'org_id', 'orgc_id'),
                ),
                'Attribute' => array(
                    'conditions' => $attributeConditions,
                    //'ShadowAttribute',
                    'AttributeTag' => array(
                        'Tag'
                    )
                )
            ),
        );
        if (!empty($options['metadata'])) {
            unset($params['contain']['Attribute']);
        }
        if (empty($options['metadata']) && empty($options['includeAllTags'])) {
            $params['contain']['Attribute']['AttributeTag']['Tag']['conditions']['exportable'] = 1;
        }
        if (isset($options['contain'])) {
            $params['contain'] = array_merge_recursive($params['contain'], $options['contain']);
        } else {
            $option['contain']['Event']['fields'] = array('id', 'info', 'org_id', 'orgc_id');
        }
        if (
            empty($options['metadata']) &&
            Configure::read('MISP.proposals_block_attributes') &&
            isset($options['conditions']['AND']['Attribute.to_ids']) &&
            $options['conditions']['AND']['Attribute.to_ids'] == 1
        ) {
            $this->Attribute->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id'))));
            $proposalRestriction =  array(
                    'ShadowAttribute' => array(
                            'conditions' => array(
                                    'AND' => array(
                                            'ShadowAttribute.deleted' => 0,
                                            'OR' => array(
                                                    'ShadowAttribute.proposal_to_delete' => 1,
                                                    'ShadowAttribute.to_ids' => 0
                                            )
                                    )
                            ),
                            'fields' => array('ShadowAttribute.id')
                    )
            );
            $params['contain'] = array_merge($params['contain']['Attribute'], $proposalRestriction);
        }
        if (isset($options['conditions'])) {
            $params['conditions']['AND'][] = $options['conditions'];
        }
        if (isset($options['order'])) {
            $params['order'] = $options['order'];
        }
        if (!isset($options['withAttachments'])) {
            $options['withAttachments'] = false;
        } else ($params['order'] = array());
        if (!isset($options['enforceWarninglist'])) {
            $options['enforceWarninglist'] = false;
        }
        if (empty($options['metadata']) && (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted'])) {
            $params['contain']['Attribute']['conditions']['AND']['Attribute.deleted'] = 0;
        }
        if (isset($options['group'])) {
            $params['group'] = array_merge(array('Object.id'), $options['group']);
        }
        if (isset($options['limit'])) {
            $params['limit'] = $options['limit'];
            if (isset($options['page'])) {
                $params['page'] = $options['page'];
            }
        }
        $results = $this->find('all', $params);
        if ($options['enforceWarninglist']) {
            $this->Warninglist = ClassRegistry::init('Warninglist');
            $warninglists = $this->Warninglist->fetchForEventView();
        }
        $results = array_values($results);
        $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
        if (empty($options['metadata'])) {
            foreach ($results as $key => $object) {
                foreach ($object['Attribute'] as $key2 => $attribute) {
                    if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'], $this->Warninglist)) {
                        unset($results[$key][$key2]);
                        continue;
                    }
                    if ($proposals_block_attributes) {
                        if (!empty($attribute['ShadowAttribute'])) {
                            unset($results[$key][$key2]);
                        } else {
                            unset($results[$key][$key2]['ShadowAttribute']);
                        }
                    }
                    if ($options['withAttachments']) {
                        if ($this->Attribute->typeIsAttachment($attribute['type'])) {
                            $encodedFile = $this->Attribute->base64EncodeAttachment($attribute);
                            $results[$key]['Attribute'][$key2]['data'] = $encodedFile;
                        }
                    }
                }
            }
        }
        return $results;
    }

    /*
     * Prepare the template form view's data, setting defaults, sorting elements
     */
    public function prepareTemplate($template, $request = array())
    {
        $temp = array();
        usort($template['ObjectTemplateElement'], function ($a, $b) {
            return $a['ui-priority'] < $b['ui-priority'];
        });
        $request_rearranged = array();
        $template_object_elements = $template['ObjectTemplateElement'];
        unset($template['ObjectTemplateElement']);
        if (!empty($request['Attribute'])) {
            foreach ($request['Attribute'] as $attribute) {
                $request_rearranged[$attribute['object_relation']][] = $attribute;
            }
{'fraction_non_alphanumeric': 0.16294430992736078, 'fraction_numerical': 0.0026343825665859563, 'mean_word_length': 1.6244725738396624, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 643, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1843749', 'n_tokens_mistral': 13462, 'n_tokens_neox': 12945, 'n_words': 2232}