<commit_before><?php if ( ! defined('BASEPATH')) exit('No direct script access allowed');


/**
 * Core Data Source controller
 *
 *
 * @author <NAME> <<EMAIL>>
 * @see ands/datasource/_data_source
 * @package ands/datasource
 *
 */
class Data_source extends MX_Controller {

	/**
	 * index page, display the angularJS data source app view
	 * @author <NAME> <<EMAIL>>
	 * @return view
	 */
	public function index() {
		acl_enforce('REGISTRY_USER');
		$data['title'] = 'Manage My Data Sources';
		$data['scripts'] = array('ds_app');
		$data['js_lib'] = array('core', 'ands_datepicker','vocab_widget','rosearch_widget', 'angular', 'socket.io');
		$this->load->view("datasource_app", $data);
	}

	/**
	 * get a JSON presentation of a data source
	 * if there's no data source speficied, get all owned datasource
	 * @author <NAME> <<EMAIL>>
	 * @param  data_source_id $id
	 * @return json
	 */
	public function get($id=false) {
		//prepare
		acl_enforce('REGISTRY_USER');

		header('Cache-Control: no-cache, must-revalidate');
		header('Content-type: application/json');
		set_exception_handler('json_exception_handler');

		$jsonData = array();
		$jsonData['status'] = 'OK';

		$this->load->model("data_sources","ds");
		if(!$id){
			// $this->benchmark->mark('code_start');
			$dataSources = $this->ds->getOwnedDataSources();
			// $this->benchmark->mark('code_end');
			// dd($this->benchmark->elapsed_time('code_start', 'code_end'));
		} elseif ($id && $id!=null) {
			// $this->benchmark->mark('code_start');
			ds_acl_enforce($id);
			$ds = $this->ds->getByID($id);
			// Should look at updating stats
            $ds->updateStats();
			$dataSources = array();
			$dataSources[] = $ds;
			// $this->benchmark->mark('code_end');
			// dd($this->benchmark->elapsed_time('code_start', 'code_end'));
		}
		$this->load->model("registry_object/registry_objects", "ro");

		$items = array();
		foreach($dataSources as $ds){
			$item = array();
			$item['title'] = $ds->title;
			$item['id'] = $ds->id;
			$item['counts'] = array();
			foreach ($this->ro->valid_status AS $status){
				if($ds->getAttribute("count_$status")>0){
					array_push($item['counts'], array('status' => $status, 'count' =>$ds->getAttribute("count_$status"), 'name'=>readable($status)));
				}
			}
			$item['qlcounts'] = array();
			foreach ($this->ro->valid_levels AS $level){
				array_push($item['qlcounts'], array('level' => $level, 'title' => ($level==4 ? 'Gold Standard Records' : 'Quality Level '.$level), 'count' =>$ds->getAttribute("count_level_$level")));
			}
			$item['classcounts'] = array();
			foreach($this->ro->valid_classes as $class){
				if($ds->getAttribute("count_$class")>0)array_push($item['classcounts'], array('class' => $class, 'count' =>$ds->getAttribute("count_$class"),'name'=>readable($class)));
			}
			$item['key']=$ds->key;
			$item['record_owner']=$ds->record_owner;
			$item['notes']=$ds->notes;



			if($id && $ds){

				foreach($ds->attributes as $attrib=>$value){
					$item[$attrib] = $value->value;
				}

				if(isset($item['crosswalks'])) {
					// $item['crosswalks'] = json_decode($item['crosswalks'], true);
				}

				if(isset($item['harvest_date'])) {
					date_default_timezone_set('Australia/Canberra');
					$item['harvest_date'] = date( 'Y-m-d H:i:s', strtotime($item['harvest_date']));
				}

				//get harvester_method


			}

			array_push($items, $item);
		}

		if($id && $ds) {
			$logs = $ds->get_logs(0, 10, null, 'all', 'all');
			$items[0]['logs'] = $logs;
		}

		$jsonData['items'] = $items;
		$jsonData = json_encode($jsonData);
		echo $jsonData;
	}

	public function upload($id=false) {
		header('Cache-Control: no-cache, must-revalidate');
		header('Content-type: application/json');
		set_exception_handler('json_exception_handler');<commit_msg>Fixed an issue where the harvester settings is missing in the data source edit screen<commit_after><?php if ( ! defined('BASEPATH')) exit('No direct script access allowed');


/**
 * Core Data Source controller
 *
 *
 * @author <NAME> <<EMAIL>>
 * @see ands/datasource/_data_source
 * @package ands/datasource
 *
 */
class Data_source extends MX_Controller {

	/**
	 * index page, display the angularJS data source app view
	 * @author <NAME> <<EMAIL>>
	 * @return view
	 */
	public function index() {
		acl_enforce('REGISTRY_USER');
		$data['title'] = 'Manage My Data Sources';
		$data['scripts'] = array('ds_app');
		$data['js_lib'] = array('core', 'ands_datepicker','vocab_widget','rosearch_widget', 'angular', 'socket.io');
		$this->load->view("datasource_app", $data);
	}

	/**
	 * get a JSON presentation of a data source
	 * if there's no data source speficied, get all owned datasource
	 * @author <NAME> <<EMAIL>>
	 * @param  data_source_id $id
	 * @return json
	 */
	public function get($id=false) {
		//prepare
		acl_enforce('REGISTRY_USER');

		header('Cache-Control: no-cache, must-revalidate');
		header('Content-type: application/json');
		set_exception_handler('json_exception_handler');

		$jsonData = array();
		$jsonData['status'] = 'OK';

		$this->load->model("data_sources","ds");
		if(!$id){
			// $this->benchmark->mark('code_start');
			$dataSources = $this->ds->getOwnedDataSources();
			// $this->benchmark->mark('code_end');
			// dd($this->benchmark->elapsed_time('code_start', 'code_end'));
		} elseif ($id && $id!=null) {
			// $this->benchmark->mark('code_start');
			ds_acl_enforce($id);
			$ds = $this->ds->getByID($id);
			// Should look at updating stats
            $ds->updateStats();
			$dataSources = array();
			$dataSources[] = $ds;
			// $this->benchmark->mark('code_end');
			// dd($this->benchmark->elapsed_time('code_start', 'code_end'));
		}
		$this->load->model("registry_object/registry_objects", "ro");

		$items = array();
		foreach($dataSources as $ds){
			$item = array();
			$item['title'] = $ds->title;
			$item['id'] = $ds->id;
			$item['counts'] = array();
			foreach ($this->ro->valid_status AS $status){
				if($ds->getAttribute("count_$status")>0){
					array_push($item['counts'], array('status' => $status, 'count' =>$ds->getAttribute("count_$status"), 'name'=>readable($status)));
				}
			}
			$item['qlcounts'] = array();
			foreach ($this->ro->valid_levels AS $level){
				array_push($item['qlcounts'], array('level' => $level, 'title' => ($level==4 ? 'Gold Standard Records' : 'Quality Level '.$level), 'count' =>$ds->getAttribute("count_level_$level")));
			}
			$item['classcounts'] = array();
			foreach($this->ro->valid_classes as $class){
				if($ds->getAttribute("count_$class")>0)array_push($item['classcounts'], array('class' => $class, 'count' =>$ds->getAttribute("count_$class"),'name'=>readable($class)));
			}
			$item['key']=$ds->key;
			$item['record_owner']=$ds->record_owner;
			$item['notes']=$ds->notes;



			if($id && $ds){

                if ($harvester_methods = get_db_config_item('harvester_methods')) {
                    $item['harvester_methods'] = $harvester_methods;
                }

				foreach($ds->attributes as $attrib=>$value){
					$item[$attrib] = $value->value;
				}

				if(isset($item['crosswalks'])) {
					// $item['crosswalks'] = json_decode($item['crosswalks'], true);
				}

				if(isset($item['harvest_date'])) {
					date_default_timezone_set('Australia/Canberra');
					$item['harvest_date'] = date( 'Y-m-d H:i:s', strtotime($item['harvest_date']));
				}

			}

			array_push($items, $item);
		}

		if($id && $ds) {
			$logs = $ds->get_logs(0, 10, null, 'all', 'all');
			$items[0]['logs'] = $logs;
		}

		$jsonData['items'] = $items;
		$jsonData = json_encode($jsonData);
		echo $jsonData;
	}

	public function upload($id=false) {
		header('Cache-Control: no-cache, must-revalidate');
		header('Content-type: application/json');
		set_exception_handler('json_exception_handler');
{'fraction_non_alphanumeric': 0.2122574842605679, 'fraction_numerical': 0.0017987922394963381, 'mean_word_length': 4.3794056668970285, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 129, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5394442', 'n_tokens_mistral': 3018, 'n_tokens_neox': 2639, 'n_words': 474}