<commit_before><?php if ( ! defined('BASEPATH')) exit('No direct script access allowed');

class Campaign extends CI_Controller {

	function __construct()
	{
		parent::__construct();
		
		$this->load->helper('api');		
		
		// Determine the environment we're run from for debugging/output 
		if (php_sapi_name() == 'cli') {   
			if (isset($_SERVER['TERM'])) {   
				$this->environment = 'terminal';  
			} else {   
				$this->environment = 'cron';
			}   
		} else { 
			$this->environment = 'server';
		}
	   	
				
	}


	/**
	 * Index Page for this controller.
	 *
	 * Maps to the following URL
	 * 		http://example.com/index.php/welcome
	 *	- or -  
	 * 		http://example.com/index.php/welcome/index
	 *	- or -
	 * Since this controller is set as the default controller in 
	 * config/routes.php, it's displayed at http://example.com/
	 *
	 * So any other public methods not prefixed with an underscore will
	 * map to /index.php/welcome/<method_name>
	 * @see http://codeigniter.com/user_guide/general/urls.html
	 */
	public function index()
	{
		
	
	}
	
	public function convert() {
		$this->load->model('campaign_model', 'campaign');			
				
		$orgs = $this->input->get('orgs', TRUE);

		$raw_data = $this->campaign->get_datagov_json($orgs);
		
		if(!empty($raw_data)) {
		
			$json_schema = $this->campaign->datajson_schema();
			$datajson_model = $this->campaign->schema_to_model($json_schema->properties);						

			$convert = array();
			foreach ($raw_data as $ckan_data) {
				$model = clone $datajson_model;						
				$convert[] = $this->campaign->datajson_crosswalk($ckan_data, $model);
			}		
			
		    header('Content-type: application/json');
		    print json_encode($convert);		
			exit;			
			
		} else {
			return false;
		}

				

		
	}


	public function csv() {
		$this->load->model('campaign_model', 'campaign');			
				
		$orgs = $this->input->get('orgs', TRUE);

		$row_total = 100;<commit_msg>Making sure json conversion goes through all pages of API result<commit_after><?php if ( ! defined('BASEPATH')) exit('No direct script access allowed');

class Campaign extends CI_Controller {

	function __construct()
	{
		parent::__construct();
		
		$this->load->helper('api');		
		
		// Determine the environment we're run from for debugging/output 
		if (php_sapi_name() == 'cli') {   
			if (isset($_SERVER['TERM'])) {   
				$this->environment = 'terminal';  
			} else {   
				$this->environment = 'cron';
			}   
		} else { 
			$this->environment = 'server';
		}
	   	
				
	}


	/**
	 * Index Page for this controller.
	 *
	 * Maps to the following URL
	 * 		http://example.com/index.php/welcome
	 *	- or -  
	 * 		http://example.com/index.php/welcome/index
	 *	- or -
	 * Since this controller is set as the default controller in 
	 * config/routes.php, it's displayed at http://example.com/
	 *
	 * So any other public methods not prefixed with an underscore will
	 * map to /index.php/welcome/<method_name>
	 * @see http://codeigniter.com/user_guide/general/urls.html
	 */
	public function index()
	{
		
	
	}
	
	public function convert() {
		$this->load->model('campaign_model', 'campaign');			
				
		$orgs = $this->input->get('orgs', TRUE);


		$row_total = 100;
		$row_count = 0;
		
		$row_pagesize = 100;
		$raw_data = array();        	        
		        
		while($row_count < $row_total) {
			$result 	= $this->campaign->get_datagov_json($orgs, $row_pagesize, $row_count, true);
			
			if(!empty($result)) {
				$row_total = $result->result->count;
				$row_count = $row_count + $row_pagesize; 

				$raw_data = array_merge($raw_data, $result->result->results);				
			} else {
				break;
			}
			
		}		        
		        		
		if(!empty($raw_data)) {
		
			$json_schema = $this->campaign->datajson_schema();
			$datajson_model = $this->campaign->schema_to_model($json_schema->properties);						

			$convert = array();
			foreach ($raw_data as $ckan_data) {
				$model = clone $datajson_model;						
				$convert[] = $this->campaign->datajson_crosswalk($ckan_data, $model);
			}		
			
		    header('Content-type: application/json');
		    print json_encode($convert);		
			exit;			
			
		} else {
			return false;
		}

				

		
	}


	public function csv() {
		$this->load->model('campaign_model', 'campaign');			
				
		$orgs = $this->input->get('orgs', TRUE);

		$row_total = 100;
{'fraction_non_alphanumeric': 0.15493283927744325, 'fraction_numerical': 0.0030106530801296896, 'mean_word_length': 2.7523892267593397, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5378246', 'n_tokens_mistral': 1771, 'n_tokens_neox': 1402, 'n_words': 293}