<commit_before><?php
App::uses('AppModel', 'Model');

class Feed extends AppModel {

	public $actsAs = array('SysLogLogable.SysLogLogable' => array(
			'change' => 'full'
		),
		'Trim',
		'Containable'
	);

	public $belongsTo = array(
			'SharingGroup' => array(
					'className' => 'SharingGroup',
					'foreignKey' => 'sharing_group_id',
			),
			'Tag' => array(
					'className' => 'Tag',
					'foreignKey' => 'tag_id',
			)
	);

	public $validate = array(
		'url' => array( // TODO add extra validation to refuse multiple time the same url from the same org
			'rule' => array('urlOrExistingFilepath'),
			'message' => 'Please enter a valid url or file path (make sure that the choice matches the input source setting).',
		),
		'provider' => 'valueNotEmpty',
		'name' => 'valueNotEmpty',
		'event_id' => array(
			'rule' => array('numeric'),
			'message' => 'Please enter a numeric event ID or leave this field blank.',
		)
	);

	// currently we only have an internal name and a display name, but later on we can expand this with versions, default settings, etc
	public $feed_types = array(
		'misp' => array(
			'name' => 'MISP Feed'
		),
		'freetext' => array(
			'name' => 'Freetext Parsed Feed'
		),
		'csv' => array(
				'name' => 'Simple CSV Parsed Feed'
		)
	);

	public function urlOrExistingFilepath($fields) {
		$input_source = empty($this->data['Feed']['input_source']) ? 'network' : $this->data['Feed']['input_source'];
		if ($input_source == 'local') {
			if ($this->data['Feed']['source_format'] == 'misp') {
				if (!is_dir($this->data['Feed']['url'])) {
					return 'For MISP type local feeds, please specify the containing directory.';
				}
			} else {
				if (!file_exists($this->data['Feed']['url'])) {
					return 'For non-MISP type local feeds, please specify the file to be ingested.';
				}
			}
		} else {
			if (!filter_var($this->data['Feed']['url'], FILTER_VALIDATE_URL)) {
				return false;
			}
		}
		return true;
	}

	public function getFeedTypesOptions() {
		$result = array();
		foreach ($this->feed_types as $key => $value) {
			$result[$key] = $value['name'];
		}
		return $result;
	}

	// gets the event UUIDs from the feed by ID
	// returns an array with the UUIDs of events that are new or that need updating
	public function getNewEventUuids($feed, $HttpSocket) {
		$result = array();
		if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
			if (file_exists($feed['Feed']['url'] . '/manifest.json')) {
				$data = file_get_contents($feed['Feed']['url'] . '/manifest.json');
			}
		} else {
			$request = $this->__createFeedRequest();
			$uri = $feed['Feed']['url'] . '/manifest.json';
			$response = $HttpSocket->get($uri, '', $request);
			if ($response->code != 200) return 1;
			$data = $response->body;
			unset($response);
		}
		$manifest = json_decode($data, true);
		if (!$manifest) return 2;
		$this->Event = ClassRegistry::init('Event');
		$events = $this->Event->find('all', array(
			'conditions' => array(
				'Event.uuid' => array_keys($manifest),
			),
			'recursive' => -1,
			'fields' => array('Event.id', 'Event.uuid', 'Event.timestamp')
		));
		foreach ($events as $event) {
			if ($event['Event']['timestamp'] < $manifest[$event['Event']['uuid']]['timestamp']) {
				$result['edit'][] = array('uuid' => $event['Event']['uuid'], 'id' => $event['Event']['id']);
			} else {
				$this->__cleanupFile($feed, '/' . $event['Event']['uuid'] . '.json');
			}
			unset($manifest[$event['Event']['uuid']]);
		}
		if (!empty($manifest)) {
			$result['add'] = array_keys($manifest);
		}
		return $result;
	}


	public function getManifest($feed, $HttpSocket) {
		$result = array();
		$request = $this->__createFeedRequest();
		if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
			if (file_exists($feed['Feed']['url'] . '/manifest.json')) {
				$data = file_get_contents($feed['Feed']['url'] . '/manifest.json');
				if (empty($data)) return false;
			} else {
				throw new NotFoundException('Invalid file.');
			}
		} else {
			$uri = $feed['Feed']['url'] . '/manifest.json';
			$response = $HttpSocket->get($uri, '', $request);
			if ($response->code != 200) {
				return false;
			}
			$data = $response->body;
			unset($response);
		}
		try {
			$events = json_decode($data, true);
		} catch (Exception $e) {
			return false;
		}
		$events = $this->__filterEventsIndex($events, $feed);
		return $events;
	}

	public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext', $page = 1, $limit = 60, &$params = array()) {
		$result = array();
		if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
			if (file_exists($feed['Feed']['url'])) {
				$data = file_get_contents($feed['Feed']['url']);
			}<commit_msg>fix: Added empty string as default for feed data<commit_after><?php
App::uses('AppModel', 'Model');

class Feed extends AppModel {

	public $actsAs = array('SysLogLogable.SysLogLogable' => array(
			'change' => 'full'
		),
		'Trim',
		'Containable'
	);

	public $belongsTo = array(
			'SharingGroup' => array(
					'className' => 'SharingGroup',
					'foreignKey' => 'sharing_group_id',
			),
			'Tag' => array(
					'className' => 'Tag',
					'foreignKey' => 'tag_id',
			)
	);

	public $validate = array(
		'url' => array( // TODO add extra validation to refuse multiple time the same url from the same org
			'rule' => array('urlOrExistingFilepath'),
			'message' => 'Please enter a valid url or file path (make sure that the choice matches the input source setting).',
		),
		'provider' => 'valueNotEmpty',
		'name' => 'valueNotEmpty',
		'event_id' => array(
			'rule' => array('numeric'),
			'message' => 'Please enter a numeric event ID or leave this field blank.',
		)
	);

	// currently we only have an internal name and a display name, but later on we can expand this with versions, default settings, etc
	public $feed_types = array(
		'misp' => array(
			'name' => 'MISP Feed'
		),
		'freetext' => array(
			'name' => 'Freetext Parsed Feed'
		),
		'csv' => array(
				'name' => 'Simple CSV Parsed Feed'
		)
	);

	public function urlOrExistingFilepath($fields) {
		$input_source = empty($this->data['Feed']['input_source']) ? 'network' : $this->data['Feed']['input_source'];
		if ($input_source == 'local') {
			if ($this->data['Feed']['source_format'] == 'misp') {
				if (!is_dir($this->data['Feed']['url'])) {
					return 'For MISP type local feeds, please specify the containing directory.';
				}
			} else {
				if (!file_exists($this->data['Feed']['url'])) {
					return 'For non-MISP type local feeds, please specify the file to be ingested.';
				}
			}
		} else {
			if (!filter_var($this->data['Feed']['url'], FILTER_VALIDATE_URL)) {
				return false;
			}
		}
		return true;
	}

	public function getFeedTypesOptions() {
		$result = array();
		foreach ($this->feed_types as $key => $value) {
			$result[$key] = $value['name'];
		}
		return $result;
	}

	// gets the event UUIDs from the feed by ID
	// returns an array with the UUIDs of events that are new or that need updating
	public function getNewEventUuids($feed, $HttpSocket) {
		$result = array();
		if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
			if (file_exists($feed['Feed']['url'] . '/manifest.json')) {
				$data = file_get_contents($feed['Feed']['url'] . '/manifest.json');
			}
		} else {
			$request = $this->__createFeedRequest();
			$uri = $feed['Feed']['url'] . '/manifest.json';
			$response = $HttpSocket->get($uri, '', $request);
			if ($response->code != 200) return 1;
			$data = $response->body;
			unset($response);
		}
		$manifest = json_decode($data, true);
		if (!$manifest) return 2;
		$this->Event = ClassRegistry::init('Event');
		$events = $this->Event->find('all', array(
			'conditions' => array(
				'Event.uuid' => array_keys($manifest),
			),
			'recursive' => -1,
			'fields' => array('Event.id', 'Event.uuid', 'Event.timestamp')
		));
		foreach ($events as $event) {
			if ($event['Event']['timestamp'] < $manifest[$event['Event']['uuid']]['timestamp']) {
				$result['edit'][] = array('uuid' => $event['Event']['uuid'], 'id' => $event['Event']['id']);
			} else {
				$this->__cleanupFile($feed, '/' . $event['Event']['uuid'] . '.json');
			}
			unset($manifest[$event['Event']['uuid']]);
		}
		if (!empty($manifest)) {
			$result['add'] = array_keys($manifest);
		}
		return $result;
	}


	public function getManifest($feed, $HttpSocket) {
		$result = array();
		$request = $this->__createFeedRequest();
		if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
			if (file_exists($feed['Feed']['url'] . '/manifest.json')) {
				$data = file_get_contents($feed['Feed']['url'] . '/manifest.json');
				if (empty($data)) return false;
			} else {
				throw new NotFoundException('Invalid file.');
			}
		} else {
			$uri = $feed['Feed']['url'] . '/manifest.json';
			$response = $HttpSocket->get($uri, '', $request);
			if ($response->code != 200) {
				return false;
			}
			$data = $response->body;
			unset($response);
		}
		try {
			$events = json_decode($data, true);
		} catch (Exception $e) {
			return false;
		}
		$events = $this->__filterEventsIndex($events, $feed);
		return $events;
	}

	public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext', $page = 1, $limit = 60, &$params = array()) {
		$result = array();
		$data = '';
		if (isset($feed['Feed']['input_source']) && $feed['Feed']['input_source'] == 'local') {
			if (file_exists($feed['Feed']['url'])) {
				$data = file_get_contents($feed['Feed']['url']);
			}
{'fraction_non_alphanumeric': 0.20876261837860338, 'fraction_numerical': 0.002497658445207618, 'mean_word_length': 3.958720330237358, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7346409', 'n_tokens_mistral': 3696, 'n_tokens_neox': 3164, 'n_words': 767}