<commit_before><?php

// $Id$

// Copyright (c) 2002-2007, <NAME>, <NAME>, <NAME>, et. al.
// All Rights Reserved. See copyright.txt for details and a complete list of authors.
// Licensed under the GNU LESSER GENERAL PUBLIC LICENSE. See license.txt for details.

@ini_set('max_execution_time', 0); //will not work if safe_mode is on
require_once('tiki-setup.php');

if ($prefs['feature_trackers'] != 'y') {
	$smarty->assign('msg', tra('This feature is disabled').': feature_trackers');
	$smarty->display('error.tpl');
	die;
}
if ($prefs['feature_ajax'] != 'y') {
	$smarty->assign('msg', tra('This feature is disabled').': feature_ajax');
	$smarty->display('error.tpl');
	die;
}
if (!isset($_REQUEST['trackerId'])) {
	$smarty->assign('msg', tra('No tracker indicated'));
	$smarty->display('error.tpl');
	die;
}

include_once 'tiki-export_tracker_monitor.php';

$monitor_filename = $prefs['tmpDir'].'/tracker_'.$_REQUEST['trackerId'].'_monitor.json';
if (is_file($monitor_filename)) {
	$stat_array = unserialize(file_get_contents($monitor_filename));
} else {
	saveStatus(array('user' => $user, 'status' => 'init', 'msg' => 'Starting...'));
}

include_once('lib/trackers/trackerlib.php');

$tracker_info = $trklib->get_tracker($_REQUEST['trackerId']);
if (empty($tracker_info)) {
	$smarty->assign('msg', tra('No tracker indicated'));
	$smarty->assign('msg', tra('No tracker indicated'));
	die;
}
if ($t = $trklib->get_tracker_options($_REQUEST['trackerId'])) {
	$tracker_info = array_merge($tracker_info,$t);
}

$tikilib->get_perm_object($_REQUEST['trackerId'], 'tracker', $tracker_info);
if ($tiki_p_view_trackers != 'y') {
	$smarty->assign('errortype', 401);
	$smarty->assign('msg', tra('You do not have permission to use this feature'));
	$smarty->display("error.tpl");
	die;
}

$filters = array();
if (!empty($_REQUEST['listfields'])) {
	if (is_string($_REQUEST['listfields'])) {
		$filters['fieldId'] = split('[,:]', $_REQUEST['listfields']);
	} elseif (is_array($_REQUEST['listfields'])) {
		$filters['fieldId'] = $_REQUEST['listfields'];
	}
} elseif (isset($_REQUEST['which']) && $_REQUEST['which'] == 'ls') {
	$filters['or'] = array('isSearchable'=>'y', 'isTblVisible'=>'y');
} elseif (isset($_REQUEST['which']) && $_REQUEST['which'] == 'list') {
	$filters['isTblVisible'] = 'y';
}
if ($tiki_p_admin_trackers != 'y') {
	$filters['isHidden'] = array('n', 'c');
}
if ($tiki_p_tracker_view_ratings != 'y') {
	$filters['not'] = array('type'=>'s');
}
$filters['not'] = array('type'=>'h');

$fields = $trklib->list_tracker_fields($_REQUEST['trackerId'], 0, -1, 'position_asc', '', true, $filters);
$listfields = array();
foreach ($fields['data'] as $field) {
	$listfields[$field['fieldId']] = $field;
}

if (!isset($_REQUEST['which'])) {
	$_REQUEST['which'] = 'all';
}
if (!isset($_REQUEST['status'])) {
	$_REQUEST['status'] = '';
}
if (!isset($_REQUEST['initial'])) {
	$_REQUEST['initial'] = '';
}
$filterFields = '';
$values = '';
$exactValues = '';
foreach ($_REQUEST as $key =>$val) {
	if (substr($key, 0, 2) == 'f_' && !empty($val) && (!is_array($val) || !empty($val[0]))) {
		$fieldId = substr($key, 2);
		$filterFields[] = $fieldId;
		if (isset($_REQUEST["x_$fieldId"]) && $_REQUEST["x_$fieldId"] == 't' ) {
			$exactValues[] = '';
			$values[] = $val;
		} else {
			$exactValues[] = $val;
			$values[] = '';
		}
	}
}

if (isset($_REQUEST['showStatus'])) {
	$showStatus = $_REQUEST['showStatus'] == 'on'? true : false;
} else {
	$showStatus = false;
}

if (isset($_REQUEST['showItemId'])) {
	$showItemId = $_REQUEST['showItemId'] == 'on'? true : false;
} else {
	$showItemId = false;
}

if (isset($_REQUEST['showCreated'])) {
	$showCreated = $_REQUEST['showCreated'] == 'on'? true : false;
} else {
	$showCreated = false;
}

if (isset($_REQUEST['showLastModif'])) {
	$showLastModif = $_REQUEST['showLastModif'] == 'on'? true : false;
} else {
	$showLastModif = false;
}

if (isset($_REQUEST['parse'])) {
	$parse = $_REQUEST['parse'] == 'on'? true : false;
} else {
	$parse = false;
}

if (empty($_REQUEST['encoding'])) {
	$_REQUEST['encoding'] = 'ISO-8859-1';
}
$encoding = $_REQUEST['encoding'];
if (empty($_REQUEST['separator'])) {
	$_REQUEST['separator'] = ',';
}
$separator = $_REQUEST['separator'];
if (empty($_REQUEST['delimitorL'])) {
	$_REQUEST['delimitorL'] = '"';
}
$delimitorL = $_REQUEST['delimitorL'];
if (empty($_REQUEST['delimitorR'])) {
	$_REQUEST['delimitorR'] = '"';
}
$delimitorR = $_REQUEST['delimitorR'];
if (empty($_REQUEST['CR'])) {
	$_REQUEST['CR'] = '%%%';
}
$CR = $_REQUEST['CR'];

$fp = null;
$temp_filename = $prefs['tmpDir'].'/tracker_'.$_REQUEST['trackerId'].'.csv';
if ($_REQUEST['debug']) {
	$fp = fopen($temp_filename, 'w');
}

include_once 'lib/core/lib/Zend/Log.php';
include_once 'lib/core/lib/Zend/Log/Writer/Stream.php';

$writer = new Zend_Log_Writer_Stream($prefs['tmpDir'].'/tracker_export.log');
$logger = new Zend_Log($writer);

$logger->info('------------- start mem used: ' . round(memory_get_usage(true)/1024, 3));

saveStatus(array('status' => 'header', 'msg' => ''));

function write_export_header() {
	header("Content-type: text/comma-separated-values; charset:".$_REQUEST['encoding']);
	if (!empty($_REQUEST['file'])) {
		if (preg_match('/.csv$/', $_REQUEST['file'])) {
			$file = $_REQUEST['file'];
		} else {
			$file = $_REQUEST['file'].'.csv';
		}
	} else {
		$file = tra('tracker').'_'.$_REQUEST['trackerId'].'.csv';
	}
	header("Content-Disposition: attachment; filename=$file");
	header("Expires: 0");
	header("Cache-Control: must-revalidate, post-check=0,pre-check=0");
	header("Pragma: public");
}
session_write_close();

if (empty($fp)) {
	write_export_header();
}

if ($tracker_info['defaultOrderKey'] == -1)
	$sort_mode = 'lastModif';
elseif ($tracker_info['defaultOrderKey'] == -2)
	$sort_mode = 'created';
elseif ($tracker_info['defaultOrderKey'] == -3)
	$sort_mode = 'itemId';
elseif (isset($tracker_info['defaultOrderKey'])) {
	$sort_mode = 'f_'.$tracker_info['defaultOrderKey'];
} else {
	$sort_mode = 'itemId';
}
if (isset($tracker_info['defaultOrderDir'])) {
	$sort_mode.= "_".$tracker_info['defaultOrderDir'];
} else {
		$sort_mode.= "_asc";
}

function needs_separator($string) {
	return empty($string) || substr($string, strlen($string) - 1, 1) == "\n";
}

// write out header
$str = '';
if ($showItemId) {
	$str .= $delimitorL.'itemId'.$delimitorL;
}
if ($showStatus) {
	$str .= needs_separator($str) ? '' : $separator;
	$str .= $delimitorL.'status'.$delimitorL;
}
if ($showCreated) {
	$str .= needs_separator($str) ? '' : $separator;
	$str .= $delimitorL.'created'.$delimitorL;
}
if ($showLastModif) {
	$str .= needs_separator($str) ? '' : $separator;
	$str .= $delimitorL.'lastModif'.$delimitorL;
}
if (count($listfields) > 0) {
	foreach ($listfields as $field) {
		$str .= needs_separator($str) ? '' : $separator;
		$str .= $delimitorL.$field['name'].' -- '.$field['fieldId'].$delimitorR;
	}
}

$str .= "\n";
if (!empty($fp)) {
	fwrite($fp, $str);
} else {
	echo $str;
}
$str = '';

$logger->info('header done: ' . round($mem/1024, 4));
saveStatus(array('status' => 'header', 'msg' => ''));

include_once 'lib/filegals/max_upload_size.php';	// sets $memory_limit and $current_memory_usage
$mem = $mem2 = memory_get_usage(true);
// just to get the number of records...
$items = $trklib->list_items($_REQUEST['trackerId'], 0, 0, $sort_mode, $listfields, $filterFields, $values, $_REQUEST['status'], $_REQUEST['initial'], $exactValues);
$cant = $items['cant'];

if (!empty($_REQUEST['recordsOffset'])) {
	$offset = $recordsOffset = (int) $_REQUEST['recordsOffset'];
} else {
	$offset = $recordsOffset = 0;
}
if (!empty($_REQUEST['recordsMax'])) {
	$recordsMax = $_REQUEST['recordsMax'];
} else {
	$recordsMax = $cant;	// all
}
if ($recordsMax + $offset > $cant) {
	$recordsMax = $cant - $offset;
}

function calc_chunkSize ($keepFree = 1) {
	global $memory_limit, $chunkSize, $recordsMax, $recordsOffset, $offset, $fields;
	if ($memory_limit < 0) {
		$chunkSize = 5000;	// unlimited memory?
	} else {
		$chunkSize = (int)(($memory_limit - memory_get_usage(true)  - $keepFree * 1048576 ) / ((count($fields['data']) * 4096 * 2) + (($recordsMax + $recordsOffset - $offset) * 2)));	// combination of possible record size and number of records
	}
	if ($chunkSize < 10) { $chunkSize = 10; }
	if ($chunkSize > $recordsMax) { $chunkSize = $recordsMax; } // limit to request or cant
}

//calc_chunkSize($memory_limit / (3 * 1048576));
calc_chunkSize(10);

$logger->info('$chunkSize: ' . $chunkSize . '    cant: '. $cant . '     fields: ' . count($fields['data']) . '      memlimit: ' . $memory_limit);
$starttime = microtime(true);
$export_cant = 0; $chunks = 0;
saveStatus(array('status' => 'waiting', 'total' => $recordsMax + $offset, 'current' => $export_cant, 'msg' => ''));

while (($items = $trklib->list_items($_REQUEST['trackerId'], $offset, $chunkSize, $sort_mode, $listfields, $filterFields, $values, $_REQUEST['status'], $_REQUEST['initial'], $exactValues)) && !empty($items['data'])) {
	
	// still need to filter the fields that are view only by the admin and the item creator
	if ($tracker_info['useRatings'] == 'y') {<commit_msg>[FIX] Better low memory handling & more progress info.<commit_after><?php

// $Id$

// Copyright (c) 2002-2007, <NAME>, <NAME>, <NAME>, et. al.
// All Rights Reserved. See copyright.txt for details and a complete list of authors.
// Licensed under the GNU LESSER GENERAL PUBLIC LICENSE. See license.txt for details.

@ini_set('max_execution_time', 0); //will not work if safe_mode is on
require_once('tiki-setup.php');

if ($prefs['feature_trackers'] != 'y') {
	$smarty->assign('msg', tra('This feature is disabled').': feature_trackers');
	$smarty->display('error.tpl');
	die;
}
if ($prefs['feature_ajax'] != 'y') {
	$smarty->assign('msg', tra('This feature is disabled').': feature_ajax');
	$smarty->display('error.tpl');
	die;
}
if (!isset($_REQUEST['trackerId'])) {
	$smarty->assign('msg', tra('No tracker indicated'));
	$smarty->display('error.tpl');
	die;
}

include_once 'tiki-export_tracker_monitor.php';

$monitor_filename = $prefs['tmpDir'].'/tracker_'.$_REQUEST['trackerId'].'_monitor.json';
if (is_file($monitor_filename)) {
	$stat_array = unserialize(file_get_contents($monitor_filename));
} else {
	saveStatus(array('user' => $user, 'status' => 'init', 'msg' => 'Starting...'));
}

include_once('lib/trackers/trackerlib.php');

$tracker_info = $trklib->get_tracker($_REQUEST['trackerId']);
if (empty($tracker_info)) {
	$smarty->assign('msg', tra('No tracker indicated'));
	$smarty->assign('msg', tra('No tracker indicated'));
	die;
}
if ($t = $trklib->get_tracker_options($_REQUEST['trackerId'])) {
	$tracker_info = array_merge($tracker_info,$t);
}

$tikilib->get_perm_object($_REQUEST['trackerId'], 'tracker', $tracker_info);
if ($tiki_p_view_trackers != 'y') {
	$smarty->assign('errortype', 401);
	$smarty->assign('msg', tra('You do not have permission to use this feature'));
	$smarty->display("error.tpl");
	die;
}

$filters = array();
if (!empty($_REQUEST['listfields'])) {
	if (is_string($_REQUEST['listfields'])) {
		$filters['fieldId'] = split('[,:]', $_REQUEST['listfields']);
	} elseif (is_array($_REQUEST['listfields'])) {
		$filters['fieldId'] = $_REQUEST['listfields'];
	}
} elseif (isset($_REQUEST['which']) && $_REQUEST['which'] == 'ls') {
	$filters['or'] = array('isSearchable'=>'y', 'isTblVisible'=>'y');
} elseif (isset($_REQUEST['which']) && $_REQUEST['which'] == 'list') {
	$filters['isTblVisible'] = 'y';
}
if ($tiki_p_admin_trackers != 'y') {
	$filters['isHidden'] = array('n', 'c');
}
if ($tiki_p_tracker_view_ratings != 'y') {
	$filters['not'] = array('type'=>'s');
}
$filters['not'] = array('type'=>'h');

$fields = $trklib->list_tracker_fields($_REQUEST['trackerId'], 0, -1, 'position_asc', '', true, $filters);
$listfields = array();
foreach ($fields['data'] as $field) {
	$listfields[$field['fieldId']] = $field;
}

if (!isset($_REQUEST['which'])) {
	$_REQUEST['which'] = 'all';
}
if (!isset($_REQUEST['status'])) {
	$_REQUEST['status'] = '';
}
if (!isset($_REQUEST['initial'])) {
	$_REQUEST['initial'] = '';
}
$filterFields = '';
$values = '';
$exactValues = '';
foreach ($_REQUEST as $key =>$val) {
	if (substr($key, 0, 2) == 'f_' && !empty($val) && (!is_array($val) || !empty($val[0]))) {
		$fieldId = substr($key, 2);
		$filterFields[] = $fieldId;
		if (isset($_REQUEST["x_$fieldId"]) && $_REQUEST["x_$fieldId"] == 't' ) {
			$exactValues[] = '';
			$values[] = $val;
		} else {
			$exactValues[] = $val;
			$values[] = '';
		}
	}
}

if (isset($_REQUEST['showStatus'])) {
	$showStatus = $_REQUEST['showStatus'] == 'on'? true : false;
} else {
	$showStatus = false;
}

if (isset($_REQUEST['showItemId'])) {
	$showItemId = $_REQUEST['showItemId'] == 'on'? true : false;
} else {
	$showItemId = false;
}

if (isset($_REQUEST['showCreated'])) {
	$showCreated = $_REQUEST['showCreated'] == 'on'? true : false;
} else {
	$showCreated = false;
}

if (isset($_REQUEST['showLastModif'])) {
	$showLastModif = $_REQUEST['showLastModif'] == 'on'? true : false;
} else {
	$showLastModif = false;
}

if (isset($_REQUEST['parse'])) {
	$parse = $_REQUEST['parse'] == 'on'? true : false;
} else {
	$parse = false;
}

if (empty($_REQUEST['encoding'])) {
	$_REQUEST['encoding'] = 'ISO-8859-1';
}
$encoding = $_REQUEST['encoding'];
if (empty($_REQUEST['separator'])) {
	$_REQUEST['separator'] = ',';
}
$separator = $_REQUEST['separator'];
if (empty($_REQUEST['delimitorL'])) {
	$_REQUEST['delimitorL'] = '"';
}
$delimitorL = $_REQUEST['delimitorL'];
if (empty($_REQUEST['delimitorR'])) {
	$_REQUEST['delimitorR'] = '"';
}
$delimitorR = $_REQUEST['delimitorR'];
if (empty($_REQUEST['CR'])) {
	$_REQUEST['CR'] = '%%%';
}
$CR = $_REQUEST['CR'];

$fp = null;
$temp_filename = $prefs['tmpDir'].'/tracker_'.$_REQUEST['trackerId'].'.csv';
if ($_REQUEST['debug']) {
	$fp = fopen($temp_filename, 'w');
}

include_once 'lib/core/lib/Zend/Log.php';
include_once 'lib/core/lib/Zend/Log/Writer/Stream.php';

$writer = new Zend_Log_Writer_Stream($prefs['tmpDir'].'/tracker_export.log');
$logger = new Zend_Log($writer);

$logger->info('------------- start mem used: ' . round(memory_get_usage(true)/1024, 3));

saveStatus(array('status' => 'header', 'msg' => '', 'current' => 0));

function write_export_header() {
	header("Content-type: text/comma-separated-values; charset:".$_REQUEST['encoding']);
	if (!empty($_REQUEST['file'])) {
		if (preg_match('/.csv$/', $_REQUEST['file'])) {
			$file = $_REQUEST['file'];
		} else {
			$file = $_REQUEST['file'].'.csv';
		}
	} else {
		$file = tra('tracker').'_'.$_REQUEST['trackerId'].'.csv';
	}
	header("Content-Disposition: attachment; filename=$file");
	header("Expires: 0");
	header("Cache-Control: must-revalidate, post-check=0,pre-check=0");
	header("Pragma: public");
}
session_write_close();

if (empty($fp)) {
	write_export_header();
}

if ($tracker_info['defaultOrderKey'] == -1)
	$sort_mode = 'lastModif';
elseif ($tracker_info['defaultOrderKey'] == -2)
	$sort_mode = 'created';
elseif ($tracker_info['defaultOrderKey'] == -3)
	$sort_mode = 'itemId';
elseif (isset($tracker_info['defaultOrderKey'])) {
	$sort_mode = 'f_'.$tracker_info['defaultOrderKey'];
} else {
	$sort_mode = 'itemId';
}
if (isset($tracker_info['defaultOrderDir'])) {
	$sort_mode.= "_".$tracker_info['defaultOrderDir'];
} else {
		$sort_mode.= "_asc";
}

function needs_separator($string) {
	return empty($string) || substr($string, strlen($string) - 1, 1) == "\n";
}

// write out header
$str = '';
if ($showItemId) {
	$str .= $delimitorL.'itemId'.$delimitorL;
}
if ($showStatus) {
	$str .= needs_separator($str) ? '' : $separator;
	$str .= $delimitorL.'status'.$delimitorL;
}
if ($showCreated) {
	$str .= needs_separator($str) ? '' : $separator;
	$str .= $delimitorL.'created'.$delimitorL;
}
if ($showLastModif) {
	$str .= needs_separator($str) ? '' : $separator;
	$str .= $delimitorL.'lastModif'.$delimitorL;
}
if (count($listfields) > 0) {
	foreach ($listfields as $field) {
		$str .= needs_separator($str) ? '' : $separator;
		$str .= $delimitorL.$field['name'].' -- '.$field['fieldId'].$delimitorR;
	}
}

$str .= "\n";
if (!empty($fp)) {
	fwrite($fp, $str);
} else {
	echo $str;
}
$str = '';

$logger->info('header done: ' . round($mem/1024, 4));
saveStatus(array('status' => 'header', 'msg' => ''));

include_once 'lib/filegals/max_upload_size.php';	// sets $memory_limit and $current_memory_usage
$mem = $mem2 = memory_get_usage(true);
// just to get the number of records...
$items = $trklib->list_items($_REQUEST['trackerId'], 0, 0, $sort_mode, $listfields, $filterFields, $values, $_REQUEST['status'], $_REQUEST['initial'], $exactValues);
$cant = $items['cant'];

if (!empty($_REQUEST['recordsOffset'])) {
	$offset = $recordsOffset = (int) $_REQUEST['recordsOffset'];
} else {
	$offset = $recordsOffset = 0;
}
if (!empty($_REQUEST['recordsMax'])) {
	$recordsMax = $_REQUEST['recordsMax'];
} else {
	$recordsMax = $cant;	// all
}
if ($recordsMax + $offset > $cant) {
	$recordsMax = $cant - $offset;
}

function calc_chunkSize ($keepFree = 1) {
	global $memory_limit, $chunkSize, $recordsMax, $recordsOffset, $offset, $fields;
	if ($memory_limit < 0) {
		$chunkSize = 5000;	// unlimited memory?
	} else {
		$freeMem = $memory_limit - memory_get_usage(true); 
		if ($freeMem < $keepFree * 1048576) {
			$keepFree = $freeMem / (1048576 * 4);	// leave 25% free in low memory conditions
		}
		$chunkSize = (int)(($freeMem - $keepFree * 1048576 ) / ((count($fields['data']) * 4096 * 6) + (($recordsMax + $recordsOffset - $offset) * 2)));	// combination of possible record size and number of records
	}
	if ($chunkSize < 10) { $chunkSize = 10; }
	if ($chunkSize > 200) { $chunkSize = 200; }
	if ($chunkSize > $recordsMax) { $chunkSize = $recordsMax; } // limit to request or cant
}

//calc_chunkSize($memory_limit / (3 * 1048576));
calc_chunkSize(10);

$logger->info('$chunkSize: ' . $chunkSize . '    cant: '. $cant . '     fields: ' . count($fields['data']) . '      memlimit: ' . $memory_limit);
$starttime = microtime(true);
$export_cant = 0; $chunks = 0;
saveStatus(array('status' => 'waiting', 'total' => $recordsMax + $offset, 'current' => $export_cant, 'msg' => ''));

while (($items = $trklib->list_items($_REQUEST['trackerId'], $offset, $chunkSize, $sort_mode, $listfields, $filterFields, $values, $_REQUEST['status'], $_REQUEST['initial'], $exactValues)) && !empty($items['data'])) {
	
	// still need to filter the fields that are view only by the admin and the item creator
	if ($tracker_info['useRatings'] == 'y') {
{'fraction_non_alphanumeric': 0.2333748173800119, 'fraction_numerical': 0.010064390455061956, 'mean_word_length': 5.645810859403093, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6411317', 'n_tokens_mistral': 6888, 'n_tokens_neox': 6589, 'n_words': 1283}