<commit_before><?php
// (c) Copyright 2002-2016 by authors of the Tiki Wiki CMS Groupware Project
//
// All Rights Reserved. See copyright.txt for details and a complete list of authors.
// Licensed under the GNU LESSER GENERAL PUBLIC LICENSE. See license.txt for details.
// $Id$

// ** This is the main script to release Tiki **
//
// To get the Tiki release HOWTO, try:
//    php doc/devtools/release.php --howto
//
// You can also get a detailed help on this script with:
//    php doc/devtools/release.php --help
//

define('TOOLS', dirname(__FILE__));
define('ROOT', realpath(TOOLS . '/../..'));
define('TEMP_DIR', 'temp');

define('CHANGELOG_FILENAME', 'changelog.txt');
define('CHANGELOG', ROOT . '/' . CHANGELOG_FILENAME);
define('COPYRIGHTS_FILENAME', 'copyright.txt');
define('COPYRIGHTS', ROOT . '/' . COPYRIGHTS_FILENAME);
define('SF_TW_MEMBERS_URL', 'http://sourceforge.net/p/tikiwiki/_members');
define('DEV_TW_MEMBERS_URL', 'http://dev.tiki.org/getTikiUser.php');
define('README_FILENAME', 'README');
define('README', ROOT . '/' . README_FILENAME);
define('LICENSE_FILENAME', 'license.txt');

// Display all errors and warnings, including strict level
define('ERROR_REPORTING_LEVEL', E_ALL | E_STRICT);
error_reporting(ERROR_REPORTING_LEVEL);

chdir(ROOT .'/');

require_once ROOT . '/lib/setup/third_party.php';
require_once TOOLS . '/svntools.php';

if (version_compare(PHP_VERSION, '5.0.0', '<')) {
	error("You need PHP version 5 or more to run this script\n");
}
$phpCommand = isset($_SERVER['_']) ? $_SERVER['_'] : 'php';
$phpCommandArguments = implode(' ', $_SERVER['argv']);

if (! ($options = get_options()) || $options['help']) {
	display_usage();
}

if ($options['howto']) {
	display_howto();
}

if (! check_svn_version()) {
	error("You need the subversion 'svn' program at least at version " . SVN_MIN_VERSION . "\n");
}

if (! $options['no-check-svn'] && has_uncommited_changes('.')) {
	error("Uncommited changes exist in the working folder.\n");
}

include_once('lib/setup/twversion.class.php');
$TWV = new TWVersion();

if ($options['only-secdb']) {
	updateSecdb($TWV->version);
	exit;
}

$script = $_SERVER['argv'][0];
$version = isset($_SERVER['argv'][1]) ? $_SERVER['argv'][1] : '';
$subrelease = isset($_SERVER['argv'][2]) ? $_SERVER['argv'][2] : '';

if (! preg_match("/^\d+\.\d+$/", $version)) {
	error("Version number should be in X.X format.\n");
}

$isPre = strpos($subrelease, 'pre') === 0;
if ($isPre) {
	$subrelease = substr($subrelease, 3);
	$pre = 'pre';
} else {
	$pre = '';
}
$splitedversion = explode('.', $version);
$mainversion = $splitedversion[0];

$check_version = $version.$subrelease;
if ($TWV->version != $check_version) {
	error("The version in the code ".strtolower($TWV->version)." differs from the version provided to the script $check_version.\nThe version should be modified in lib/setup/twversion.class.php to match the released version.");
}

echo color("\nTiki release process started for version '$version" . ($subrelease ? " $subrelease" : '') . "'\n", 'cyan');
if ($isPre) {
	echo color("The script is running in 'pre-release' mode, which means that no subversion tag will be created.\n", 'yellow');
}

if (! $options['no-first-update'] && important_step('Update working copy to the last revision')) {
	echo "Update in progress...";
	update_working_copy('.');

	if (! $options['no-check-svn'] && has_uncommited_changes('.')) {
		error("\rUncommited changes exist in the working folder.\n");
	}
	$revision = (int) get_info('.')->entry->commit['revision'];
	info("\r>> Checkout updated to revision $revision.");
}

if (empty($subrelease)) {
	$branch = "branches/$mainversion.x";
	$tag = "tags/$version";
	$packageVersion = $version;
	if (! empty($pre)) {
		$packageVersion .= ".$pre";
	}
	$secdbVersion = $version;
} else {
	$branch = "branches/$mainversion.x";
	$tag = "tags/$version$subrelease";
	$packageVersion = "$version.$pre$subrelease";
	$secdbVersion = "$version$subrelease";
}

if (! $options['no-readme-update'] && important_step("Update '" . README_FILENAME . "' file")) {
	update_readme_file($secdbVersion, $version);
	info('>> ' . README_FILENAME . ' file updated.');
	important_step('Commit updated ' . README_FILENAME . ' file', true, "[REL] Update " . README_FILENAME . " file for $secdbVersion");
}

if (! $options['no-lang-update'] && important_step("Update language files")) {
	passthru("$phpCommand get_strings.php");
	$removeFiles = glob('lang/*/language.php.old');
	foreach ($removeFiles as $rf) {
		unlink($rf);
	}
	unset($removeFiles);
	info('>> Language files updated and temporary files removed.');
	important_step('Commit updated language files', true, "[REL] Update language.php files for $secdbVersion");
}

if (! $options['no-changelog-update'] && important_step("Update '" . CHANGELOG_FILENAME . "' file (using final version number '$version')")) {
	if ($ucf = update_changelog_file($version)) {
		if ($ucf['nbCommits'] == 0) {
			info('>> Changelog updated (last commits were already inside)');
		} else {
			if ($ucf['sameFinalVersion']) {
				info(">> There were already some commits for the same final version number in the changelog. Merging them with the new ones.");
			}
			info(">> Changelog updated with {$ucf['nbCommits']} new commits (revision {$ucf['firstRevision']} to {$ucf['lastRevision']}), excluding duplicates, merges and release-related commits.");
		}
		important_step("Commit new " . CHANGELOG_FILENAME, true, "[REL] Update " . CHANGELOG_FILENAME . " for $secdbVersion");
	} else {
		error('Changelog update failed.');
	}
	unset($ucf);
}

$nbCommiters = 0;
if (! $options['no-copyright-update'] && important_step("Update '" . COPYRIGHTS_FILENAME . "' file (using final version number '$version')")) {
	if ($ucf = update_copyright_file($mainversion . '.0')) {
		info(
			"\r>> Copyrights updated: "
			. ($ucf['newContributors'] == 0 ? 'No new contributor, ' : "+{$ucf['newContributors']} contributor(s), ")
			. ($ucf['newCommits'] == 0 ? 'No new commit' : "+{$ucf['newCommits']} commit(s)")
		);
		important_step("Commit new " . COPYRIGHTS_FILENAME, true, "[REL] Update " . COPYRIGHTS_FILENAME . " for $secdbVersion");
	} else {
		error('Copyrights update failed.');
	}
}

if (! $options['no-check-php'] && important_step("Check syntax of all PHP files")) {
	$error_msg = '';
	$dir = '.';
	check_php_syntax($dir, $error_msg, $options['no-check-php-warnings']) or error($error_msg);
	info('>> Current PHP code successfully passed the syntax check.');
}

if (! $options['no-check-smarty'] && important_step("Check syntax of all Smarty templates")) {
	$error_msg = '';
	check_smarty_syntax($error_msg);
	info('>> Current Smarty code successfully passed the syntax check.');
}

if (! $options['no-secdb'] && important_step("Update SecDB file(s) 'db/tiki-secdb_{$version}_mysql.sql'")) {
	if(updateSecdb($TWV->version))
		important_step("Commit SecDB file changes", true, "[REL] SecDB for $secdbVersion");
}

if ($isPre) {
	if (! $options['no-packaging'] && important_step("Build packages files")) {
		build_packages($packageVersion);
		echo color("\nMake sure these tarballs are tested by at least 3 different people.\n\n", 'cyan');
	} else {
		echo color("This was the last step.\n", 'cyan');
	}
} else {


	if (! $options['no-tagging']) {
		$fb = full($branch);
		$ft = full($tag);

		$tagAlreadyExists = isset(get_info($ft)->entry);
		if ($tagAlreadyExists && important_step("The Tag '$tag' already exists: Delete the existing tag in order to create a new one")) {
			$commit_msg = "[REL] Deleting tag '$tag' in order to create a new one";
			if ($options['no-commit']) {
				print "Skipping actual commit ('$commit_msg') because no-commit = true\n";
			} else {
				`svn rm $ft -m "$commit_msg"`;
				$tagAlreadyExists = false;
				info(">> Tag '$tag' deleted.");
			}
		}
		if (! $tagAlreadyExists) {
			update_working_copy('.');
			$revision = (int) get_info(ROOT)->entry->commit['revision'];
			if (important_step("Tag release using branch '$branch' at revision $revision")) {
				$commit_msg = '[REL] Tagging release';
				if ($options['no-commit']) {
					print "Skipping actual commit ('$commit_msg') because no-commit = true\n";
				} else {
					`svn copy $fb -r$revision $ft -m "$commit_msg"`;
					info(">> Tag '$tag' created.");
				}
			}
		}
	}

	if (! $options['no-packaging'] && important_step("Build packages files")) {
		build_packages($packageVersion);
		echo color("\nUpload the files on SourceForge.\nInstructions can be found here: http://sourceforge.net/p/forge/documentation/Files/\n", 'cyan');
		echo color("\nAlternately, do it by hand after replacing SF_LOGIN with your sf.net login\nand after replacing PATH according to https://sourceforge.net/projects/tikiwiki/files/\ncd ~/tikipack/$packageVersion; scp tiki-$packageVersion.* <EMAIL>:/home/frs/project/t/ti/tikiwiki/Tiki_PATH/$packageVersion \n\n", 'cyan');
	} else {
		echo color("This was the last step.\n", 'cyan');
	}
}

// Helper functions


/**
 *
 * Will remove old secdb files and generate a new one based on working copy files.
 *
 * @param $version string The current tiki version to use eg. 17.0 or 21.0RC1
 * @return bool true on success
 */

function updateSecdb($version)
{
	// first unset any preexisting files.
	echo (">>");
	$svn = preg_match('/svn$/', $version);

	// if we are not creating a release skip deleting old files.
	if (!$svn) {
	$files = glob(ROOT . '/db/tiki-secdb_*_mysql.sql');
	foreach ($files as $file) {
			$file = escapeshellarg($file);
			`svn delete $file --force`;
		}
		echo (' Removed '.count($files).' old secdb files.');
	}

	$file = "/db/tiki-secdb_{$version}_mysql.sql";

	if (!$fp = @fopen(ROOT.$file, 'w')){
		error('The SecDB file "' . ROOT.$file . '" is not writable or can\'t be created.');
		return false;
	}
	$queries = array();
	md5_check_dir(ROOT, $version, $queries);

	if (! empty($queries)) {
		sort($queries);
		fwrite($fp, "start transaction;\n");
		fwrite($fp, "DELETE FROM `tiki_secdb` WHERE `tiki_version` = '$version';\n\n");
		foreach ($queries as $q) {
			fwrite($fp, "$q\n");
		}
		fwrite($fp, "commit;\n");
	}
	fclose($fp);

	echo (" $file was generated.\n");
	if (!$svn) {
		$file = escapeshellarg(ROOT.$file); // escape file name for use in command line.
		`svn add $file`;
	}
	return true;
}

/**
 * @param $dir
 * @param $version
 * @param $queries
 */
function md5_check_dir($dir, $version, &$queries)
{
	$d = dir($dir);
	$link = null;

	while (false !== ($e = $d->read())) {
		$entry = $dir . '/' . $e;
		if (is_dir($entry)) {
			// do not descend and no CVS/Subversion files
			if ($e != '..' && $e != '.' && $e != 'CVS' && $e != '.svn' && $entry != ROOT . '/temp' && $entry != ROOT . '/vendor_custom') {
				md5_check_dir($entry, $version, $queries);
			}
		} else {
			if (preg_match('/\.(sql|css|tpl|js|php)$/', $e) && realpath($entry) != __FILE__ && $entry != './db/local.php') {
				$file = '.' . substr($entry, strlen(ROOT));

				// Escape filename. Since this requires a connection to MySQL (due to the charset), do so conditionally to reduce the risk of connection failure. 
				if (! preg_match('/^[a-zA-Z0-9\/ _+.-]+$/', $file) ) {
					if (! $link) {
						$link = mysqli_connect();

						if (mysqli_connect_errno()) {
							global $phpCommand, $phpCommandArguments;
							error(
								"SecDB step failed because some filenames need escaping but no MySQL connection has been found (" . mysqli_connect_error() . ")."
								. "\nTry this command line instead (replace HOST, USER and PASS by a valid MySQL host, user and password) :"
								. "\n\n\t" . $phpCommand
								. " -d mysqli.default_host=HOST -d mysqli.default_user=USER -d mysqli.default_pw=PASS "
								. $phpCommandArguments . "\n"
							);
						}
					}
					$file = @mysqli_real_escape_string($link,$file);
				}

				if (is_readable($entry)) {
					$hash = md5_file($entry);
					$queries[] = "INSERT INTO `tiki_secdb` (`filename`, `md5_value`, `tiki_version`) VALUES('$file', '$hash', '$version');";
				}
			}
		}
	}
	$d->close();
}

/**
 *
 * Deletes a directory and its contents.
 *
 * @param string $dir directory to delete.
 * @return string|bool returns the filename that an error occurred in, false otherwise
 */

function rrmdir($dir) {
	if (is_dir($dir)) {
		$objects = scandir($dir);
		foreach ($objects as $object) {
			if ($object != "." && $object != "..") {
				@chmod($dir."/".$object,0777);
				if (filetype($dir."/".$object) === 'dir'){
					$error = rrmdir($dir."/".$object);
					if ($error)
						return $error;
				}else
					if (!@unlink($dir."/".$object))
						return 'Could not delete '.$dir."/".$object."\n";
			}
		}
		reset($objects);
		@unlink($dir.'/.DS_store');
		if (!@rmdir($dir)){
			return 'Could not delete '.$dir."\n";
		}
	}
	return false;
}


/**
 *
 * Recursivley deletes specific files or directories
 *
 * @param string $src Directory to search through
 * @param array $files An array of file names to delete.
 */

function removeFiles($src,$files) {
	$dir = opendir($src);
	while(false !== ( $file = readdir($dir)) ) {
		if (( $file != '.' ) && ( $file != '..' )) {
			$full = $src . '/' . $file;
			if ( is_dir($full) ) {
				$flag = false;

				foreach ($files as $delfile) {
					if (basename($full) === $delfile) {
						rrmdir($full);
						$flag =true;
						break;
					}
				}
				if (!$flag)
					removeFiles($full, $files);

			}else {
				foreach($files as $delfile) {
					if (basename($full) === $delfile) {
						@chmod($full, 0777);
						unlink($full);
						break;
					}
				}
			}
		}
	}
	closedir($dir);
}


/**
 *
 * Recursively sets permissions. Files get 775 and directories 664.
 *
 * @param string $src The directory to set permissions for
 */

function setPermissions($src) {
	$dir = opendir($src);
	while(false !== ( $file = readdir($dir)) ) {
		if (( $file != '.' ) && ( $file != '..' )) {
			$full = $src . '/' . $file;
			if ( is_dir($full) ) {
				setPermissions($full);
				chmod($full,0755
				);
			}
			else {
				chmod($full,0664);

			}
		}
	}
	closedir($dir);
}


/**
 *
 * Prepares and generates the release packages.
 *
 * @param string $releaseVersion	Version of tiki that is being released.
 */

function build_packages($releaseVersion)
{
	global $options;

	$workDir=$_SERVER['HOME']."/tikipack";
	$relDir = $workDir. '/tiki-' .$releaseVersion;

	echo  "Seting up ~/tikipack directory\n";
	if (!is_dir($workDir)) {
		if (!mkdir($workDir)) {
			error('Cant make ' . $workDir."\n");
			die();
		}
	}

	// remove previous files if they exist.
	if (is_dir($relDir)) {
		echo "Removing previous files\n";
		$shellout = rrmdir($relDir);
		if ($shellout)
			die ($shellout."\n");
	}
	$removeFiles = glob($relDir.'.*');
	foreach ($removeFiles as $file){
		unlink($file);
	}

	// create a export in tikipack to work with
	echo "Creating SVN export from working copy\n";
	$shellout = shell_exec('svn export '.escapeshellarg(ROOT).' '.escapeshellarg($relDir).' 2>&1');
	if ($options['debug-packaging'])
		echo $shellout."\n";


	if (!is_file($relDir.'/vendor_bundled/composer.json')){
		echo 'composer.json not found. Aborting.'."\n";
		die();
	}

	if (is_file($workDir.'/composer.phar')){
		if (!unlink($workDir.'/composer.phar')){
			echo "Can't delete tikipack/composer.phar. Aborting."."\n";
			die();
		}
	}

	echo "Downloading composer.phar"."\n";
	if (!file_put_contents($workDir.'/composer.phar', file_get_contents('http://getcomposer.org/composer.phar'))){
		echo "Can't create tikipack/composer.phar. Aborting."."\n";
		die();
	}

	echo 'Installing dependencies through composer'."\n";
	$shellout =  shell_exec('php '.escapeshellarg($workDir.'/composer.phar').' install -d '.escapeshellarg($relDir.'/vendor_bundled').' --prefer-dist --no-dev 2>&1');
	if ($options['debug-packaging'])
		echo $shellout."\n";

	if (strpos($shellout,'Fatal error:') || strpos($shellout,'Installation failed,')){
		echo 'Composer Instillation Failed. Exiting'."\n";
		die();
	}
	if ($options['debug-packaging'])
		echo $shellout."\n";

	echo "Removing development files\n";
	$shellout = rrmdir($relDir.'/tests');
	if ($shellout)
		die ($shellout."\n");

	$shellout = rrmdir($relDir.'/db/convertscripts');
	if ($shellout)
		die ($shellout."\n");

	$shellout = rrmdir($relDir.'/doc/devtools');
	if ($shellout)
		die ($shellout."\n");

	removeFiles($relDir,array('.cvsignore','.svnignore','.gitignore','.gitkeep','.eslintignore','.eslintrc','.npmignore','.editorconfig','.hound.yml','.travis.yml','.jshintignore','.github'));

	echo "Removing language file comments\n";
	foreach (scandir($relDir.'/lang') as $strip) {
		if (is_file($relDir . '/lang/' .$strip.'/language.php'))
			$shellout = shell_exec('php '.escapeshellarg(dirname(__FILE__).'/stripcomments.php').' '. escapeshellarg($relDir . '/lang/' .$strip.'/language.php').' 2>&1');
		if ($shellout)
			die ($shellout."\n");
	}<commit_msg>[ENH] release - remove /bin from root folder<commit_after><?php
// (c) Copyright 2002-2016 by authors of the Tiki Wiki CMS Groupware Project
//
// All Rights Reserved. See copyright.txt for details and a complete list of authors.
// Licensed under the GNU LESSER GENERAL PUBLIC LICENSE. See license.txt for details.
// $Id$

// ** This is the main script to release Tiki **
//
// To get the Tiki release HOWTO, try:
//    php doc/devtools/release.php --howto
//
// You can also get a detailed help on this script with:
//    php doc/devtools/release.php --help
//

define('TOOLS', dirname(__FILE__));
define('ROOT', realpath(TOOLS . '/../..'));
define('TEMP_DIR', 'temp');

define('CHANGELOG_FILENAME', 'changelog.txt');
define('CHANGELOG', ROOT . '/' . CHANGELOG_FILENAME);
define('COPYRIGHTS_FILENAME', 'copyright.txt');
define('COPYRIGHTS', ROOT . '/' . COPYRIGHTS_FILENAME);
define('SF_TW_MEMBERS_URL', 'http://sourceforge.net/p/tikiwiki/_members');
define('DEV_TW_MEMBERS_URL', 'http://dev.tiki.org/getTikiUser.php');
define('README_FILENAME', 'README');
define('README', ROOT . '/' . README_FILENAME);
define('LICENSE_FILENAME', 'license.txt');

// Display all errors and warnings, including strict level
define('ERROR_REPORTING_LEVEL', E_ALL | E_STRICT);
error_reporting(ERROR_REPORTING_LEVEL);

chdir(ROOT .'/');

require_once ROOT . '/lib/setup/third_party.php';
require_once TOOLS . '/svntools.php';

if (version_compare(PHP_VERSION, '5.0.0', '<')) {
	error("You need PHP version 5 or more to run this script\n");
}
$phpCommand = isset($_SERVER['_']) ? $_SERVER['_'] : 'php';
$phpCommandArguments = implode(' ', $_SERVER['argv']);

if (! ($options = get_options()) || $options['help']) {
	display_usage();
}

if ($options['howto']) {
	display_howto();
}

if (! check_svn_version()) {
	error("You need the subversion 'svn' program at least at version " . SVN_MIN_VERSION . "\n");
}

if (! $options['no-check-svn'] && has_uncommited_changes('.')) {
	error("Uncommited changes exist in the working folder.\n");
}

include_once('lib/setup/twversion.class.php');
$TWV = new TWVersion();

if ($options['only-secdb']) {
	updateSecdb($TWV->version);
	exit;
}

$script = $_SERVER['argv'][0];
$version = isset($_SERVER['argv'][1]) ? $_SERVER['argv'][1] : '';
$subrelease = isset($_SERVER['argv'][2]) ? $_SERVER['argv'][2] : '';

if (! preg_match("/^\d+\.\d+$/", $version)) {
	error("Version number should be in X.X format.\n");
}

$isPre = strpos($subrelease, 'pre') === 0;
if ($isPre) {
	$subrelease = substr($subrelease, 3);
	$pre = 'pre';
} else {
	$pre = '';
}
$splitedversion = explode('.', $version);
$mainversion = $splitedversion[0];

$check_version = $version.$subrelease;
if ($TWV->version != $check_version) {
	error("The version in the code ".strtolower($TWV->version)." differs from the version provided to the script $check_version.\nThe version should be modified in lib/setup/twversion.class.php to match the released version.");
}

echo color("\nTiki release process started for version '$version" . ($subrelease ? " $subrelease" : '') . "'\n", 'cyan');
if ($isPre) {
	echo color("The script is running in 'pre-release' mode, which means that no subversion tag will be created.\n", 'yellow');
}

if (! $options['no-first-update'] && important_step('Update working copy to the last revision')) {
	echo "Update in progress...";
	update_working_copy('.');

	if (! $options['no-check-svn'] && has_uncommited_changes('.')) {
		error("\rUncommited changes exist in the working folder.\n");
	}
	$revision = (int) get_info('.')->entry->commit['revision'];
	info("\r>> Checkout updated to revision $revision.");
}

if (empty($subrelease)) {
	$branch = "branches/$mainversion.x";
	$tag = "tags/$version";
	$packageVersion = $version;
	if (! empty($pre)) {
		$packageVersion .= ".$pre";
	}
	$secdbVersion = $version;
} else {
	$branch = "branches/$mainversion.x";
	$tag = "tags/$version$subrelease";
	$packageVersion = "$version.$pre$subrelease";
	$secdbVersion = "$version$subrelease";
}

if (! $options['no-readme-update'] && important_step("Update '" . README_FILENAME . "' file")) {
	update_readme_file($secdbVersion, $version);
	info('>> ' . README_FILENAME . ' file updated.');
	important_step('Commit updated ' . README_FILENAME . ' file', true, "[REL] Update " . README_FILENAME . " file for $secdbVersion");
}

if (! $options['no-lang-update'] && important_step("Update language files")) {
	passthru("$phpCommand get_strings.php");
	$removeFiles = glob('lang/*/language.php.old');
	foreach ($removeFiles as $rf) {
		unlink($rf);
	}
	unset($removeFiles);
	info('>> Language files updated and temporary files removed.');
	important_step('Commit updated language files', true, "[REL] Update language.php files for $secdbVersion");
}

if (! $options['no-changelog-update'] && important_step("Update '" . CHANGELOG_FILENAME . "' file (using final version number '$version')")) {
	if ($ucf = update_changelog_file($version)) {
		if ($ucf['nbCommits'] == 0) {
			info('>> Changelog updated (last commits were already inside)');
		} else {
			if ($ucf['sameFinalVersion']) {
				info(">> There were already some commits for the same final version number in the changelog. Merging them with the new ones.");
			}
			info(">> Changelog updated with {$ucf['nbCommits']} new commits (revision {$ucf['firstRevision']} to {$ucf['lastRevision']}), excluding duplicates, merges and release-related commits.");
		}
		important_step("Commit new " . CHANGELOG_FILENAME, true, "[REL] Update " . CHANGELOG_FILENAME . " for $secdbVersion");
	} else {
		error('Changelog update failed.');
	}
	unset($ucf);
}

$nbCommiters = 0;
if (! $options['no-copyright-update'] && important_step("Update '" . COPYRIGHTS_FILENAME . "' file (using final version number '$version')")) {
	if ($ucf = update_copyright_file($mainversion . '.0')) {
		info(
			"\r>> Copyrights updated: "
			. ($ucf['newContributors'] == 0 ? 'No new contributor, ' : "+{$ucf['newContributors']} contributor(s), ")
			. ($ucf['newCommits'] == 0 ? 'No new commit' : "+{$ucf['newCommits']} commit(s)")
		);
		important_step("Commit new " . COPYRIGHTS_FILENAME, true, "[REL] Update " . COPYRIGHTS_FILENAME . " for $secdbVersion");
	} else {
		error('Copyrights update failed.');
	}
}

if (! $options['no-check-php'] && important_step("Check syntax of all PHP files")) {
	$error_msg = '';
	$dir = '.';
	check_php_syntax($dir, $error_msg, $options['no-check-php-warnings']) or error($error_msg);
	info('>> Current PHP code successfully passed the syntax check.');
}

if (! $options['no-check-smarty'] && important_step("Check syntax of all Smarty templates")) {
	$error_msg = '';
	check_smarty_syntax($error_msg);
	info('>> Current Smarty code successfully passed the syntax check.');
}

if (! $options['no-secdb'] && important_step("Update SecDB file(s) 'db/tiki-secdb_{$version}_mysql.sql'")) {
	if(updateSecdb($TWV->version))
		important_step("Commit SecDB file changes", true, "[REL] SecDB for $secdbVersion");
}

if ($isPre) {
	if (! $options['no-packaging'] && important_step("Build packages files")) {
		build_packages($packageVersion);
		echo color("\nMake sure these tarballs are tested by at least 3 different people.\n\n", 'cyan');
	} else {
		echo color("This was the last step.\n", 'cyan');
	}
} else {


	if (! $options['no-tagging']) {
		$fb = full($branch);
		$ft = full($tag);

		$tagAlreadyExists = isset(get_info($ft)->entry);
		if ($tagAlreadyExists && important_step("The Tag '$tag' already exists: Delete the existing tag in order to create a new one")) {
			$commit_msg = "[REL] Deleting tag '$tag' in order to create a new one";
			if ($options['no-commit']) {
				print "Skipping actual commit ('$commit_msg') because no-commit = true\n";
			} else {
				`svn rm $ft -m "$commit_msg"`;
				$tagAlreadyExists = false;
				info(">> Tag '$tag' deleted.");
			}
		}
		if (! $tagAlreadyExists) {
			update_working_copy('.');
			$revision = (int) get_info(ROOT)->entry->commit['revision'];
			if (important_step("Tag release using branch '$branch' at revision $revision")) {
				$commit_msg = '[REL] Tagging release';
				if ($options['no-commit']) {
					print "Skipping actual commit ('$commit_msg') because no-commit = true\n";
				} else {
					`svn copy $fb -r$revision $ft -m "$commit_msg"`;
					info(">> Tag '$tag' created.");
				}
			}
		}
	}

	if (! $options['no-packaging'] && important_step("Build packages files")) {
		build_packages($packageVersion);
		echo color("\nUpload the files on SourceForge.\nInstructions can be found here: http://sourceforge.net/p/forge/documentation/Files/\n", 'cyan');
		echo color("\nAlternately, do it by hand after replacing SF_LOGIN with your sf.net login\nand after replacing PATH according to https://sourceforge.net/projects/tikiwiki/files/\ncd ~/tikipack/$packageVersion; scp tiki-$packageVersion.* SF_LOGIN<EMAIL>:/home/frs/project/t/ti/tikiwiki/Tiki_PATH/$packageVersion \n\n", 'cyan');
	} else {
		echo color("This was the last step.\n", 'cyan');
	}
}

// Helper functions


/**
 *
 * Will remove old secdb files and generate a new one based on working copy files.
 *
 * @param $version string The current tiki version to use eg. 17.0 or 21.0RC1
 * @return bool true on success
 */

function updateSecdb($version)
{
	// first unset any preexisting files.
	echo (">>");
	$svn = preg_match('/svn$/', $version);

	// if we are not creating a release skip deleting old files.
	if (!$svn) {
	$files = glob(ROOT . '/db/tiki-secdb_*_mysql.sql');
	foreach ($files as $file) {
			$file = escapeshellarg($file);
			`svn delete $file --force`;
		}
		echo (' Removed '.count($files).' old secdb files.');
	}

	$file = "/db/tiki-secdb_{$version}_mysql.sql";

	if (!$fp = @fopen(ROOT.$file, 'w')){
		error('The SecDB file "' . ROOT.$file . '" is not writable or can\'t be created.');
		return false;
	}
	$queries = array();
	md5_check_dir(ROOT, $version, $queries);

	if (! empty($queries)) {
		sort($queries);
		fwrite($fp, "start transaction;\n");
		fwrite($fp, "DELETE FROM `tiki_secdb` WHERE `tiki_version` = '$version';\n\n");
		foreach ($queries as $q) {
			fwrite($fp, "$q\n");
		}
		fwrite($fp, "commit;\n");
	}
	fclose($fp);

	echo (" $file was generated.\n");
	if (!$svn) {
		$file = escapeshellarg(ROOT.$file); // escape file name for use in command line.
		`svn add $file`;
	}
	return true;
}

/**
 * @param $dir
 * @param $version
 * @param $queries
 */
function md5_check_dir($dir, $version, &$queries)
{
	$d = dir($dir);
	$link = null;

	while (false !== ($e = $d->read())) {
		$entry = $dir . '/' . $e;
		if (is_dir($entry)) {
			// do not descend and no CVS/Subversion files
			if ($e != '..' && $e != '.' && $e != 'CVS' && $e != '.svn' && $entry != ROOT . '/temp' && $entry != ROOT . '/vendor_custom') {
				md5_check_dir($entry, $version, $queries);
			}
		} else {
			if (preg_match('/\.(sql|css|tpl|js|php)$/', $e) && realpath($entry) != __FILE__ && $entry != './db/local.php') {
				$file = '.' . substr($entry, strlen(ROOT));

				// Escape filename. Since this requires a connection to MySQL (due to the charset), do so conditionally to reduce the risk of connection failure. 
				if (! preg_match('/^[a-zA-Z0-9\/ _+.-]+$/', $file) ) {
					if (! $link) {
						$link = mysqli_connect();

						if (mysqli_connect_errno()) {
							global $phpCommand, $phpCommandArguments;
							error(
								"SecDB step failed because some filenames need escaping but no MySQL connection has been found (" . mysqli_connect_error() . ")."
								. "\nTry this command line instead (replace HOST, USER and PASS by a valid MySQL host, user and password) :"
								. "\n\n\t" . $phpCommand
								. " -d mysqli.default_host=HOST -d mysqli.default_user=USER -d mysqli.default_pw=PASS "
								. $phpCommandArguments . "\n"
							);
						}
					}
					$file = @mysqli_real_escape_string($link,$file);
				}

				if (is_readable($entry)) {
					$hash = md5_file($entry);
					$queries[] = "INSERT INTO `tiki_secdb` (`filename`, `md5_value`, `tiki_version`) VALUES('$file', '$hash', '$version');";
				}
			}
		}
	}
	$d->close();
}

/**
 *
 * Deletes a directory and its contents.
 *
 * @param string $dir directory to delete.
 * @return string|bool returns the filename that an error occurred in, false otherwise
 */

function rrmdir($dir) {
	if (is_dir($dir)) {
		$objects = scandir($dir);
		foreach ($objects as $object) {
			if ($object != "." && $object != "..") {
				@chmod($dir."/".$object,0777);
				if (filetype($dir."/".$object) === 'dir'){
					$error = rrmdir($dir."/".$object);
					if ($error)
						return $error;
				}else
					if (!@unlink($dir."/".$object))
						return 'Could not delete '.$dir."/".$object."\n";
			}
		}
		reset($objects);
		@unlink($dir.'/.DS_store');
		if (!@rmdir($dir)){
			return 'Could not delete '.$dir."\n";
		}
	}
	return false;
}


/**
 *
 * Recursivley deletes specific files or directories
 *
 * @param string $src Directory to search through
 * @param array $files An array of file names to delete.
 */

function removeFiles($src,$files) {
	$dir = opendir($src);
	while(false !== ( $file = readdir($dir)) ) {
		if (( $file != '.' ) && ( $file != '..' )) {
			$full = $src . '/' . $file;
			if ( is_dir($full) ) {
				$flag = false;

				foreach ($files as $delfile) {
					if (basename($full) === $delfile) {
						rrmdir($full);
						$flag =true;
						break;
					}
				}
				if (!$flag)
					removeFiles($full, $files);

			}else {
				foreach($files as $delfile) {
					if (basename($full) === $delfile) {
						@chmod($full, 0777);
						unlink($full);
						break;
					}
				}
			}
		}
	}
	closedir($dir);
}


/**
 *
 * Recursively sets permissions. Files get 775 and directories 664.
 *
 * @param string $src The directory to set permissions for
 */

function setPermissions($src) {
	$dir = opendir($src);
	while(false !== ( $file = readdir($dir)) ) {
		if (( $file != '.' ) && ( $file != '..' )) {
			$full = $src . '/' . $file;
			if ( is_dir($full) ) {
				setPermissions($full);
				chmod($full,0755
				);
			}
			else {
				chmod($full,0664);

			}
		}
	}
	closedir($dir);
}


/**
 *
 * Prepares and generates the release packages.
 *
 * @param string $releaseVersion	Version of tiki that is being released.
 */

function build_packages($releaseVersion)
{
	global $options;

	$workDir=$_SERVER['HOME']."/tikipack";
	$relDir = $workDir. '/tiki-' .$releaseVersion;

	echo  "Seting up ~/tikipack directory\n";
	if (!is_dir($workDir)) {
		if (!mkdir($workDir)) {
			error('Cant make ' . $workDir."\n");
			die();
		}
	}

	// remove previous files if they exist.
	if (is_dir($relDir)) {
		echo "Removing previous files\n";
		$shellout = rrmdir($relDir);
		if ($shellout)
			die ($shellout."\n");
	}
	$removeFiles = glob($relDir.'.*');
	foreach ($removeFiles as $file){
		unlink($file);
	}

	// create a export in tikipack to work with
	echo "Creating SVN export from working copy\n";
	$shellout = shell_exec('svn export '.escapeshellarg(ROOT).' '.escapeshellarg($relDir).' 2>&1');
	if ($options['debug-packaging'])
		echo $shellout."\n";


	if (!is_file($relDir.'/vendor_bundled/composer.json')){
		echo 'composer.json not found. Aborting.'."\n";
		die();
	}

	if (is_file($workDir.'/composer.phar')){
		if (!unlink($workDir.'/composer.phar')){
			echo "Can't delete tikipack/composer.phar. Aborting."."\n";
			die();
		}
	}

	echo "Downloading composer.phar"."\n";
	if (!file_put_contents($workDir.'/composer.phar', file_get_contents('http://getcomposer.org/composer.phar'))){
		echo "Can't create tikipack/composer.phar. Aborting."."\n";
		die();
	}

	echo 'Installing dependencies through composer'."\n";
	$shellout =  shell_exec('php '.escapeshellarg($workDir.'/composer.phar').' install -d '.escapeshellarg($relDir.'/vendor_bundled').' --prefer-dist --no-dev 2>&1');
	if ($options['debug-packaging'])
		echo $shellout."\n";

	if (strpos($shellout,'Fatal error:') || strpos($shellout,'Installation failed,')){
		echo 'Composer Instillation Failed. Exiting'."\n";
		die();
	}
	if ($options['debug-packaging'])
		echo $shellout."\n";

	echo "Removing development files\n";
	$shellout = rrmdir($relDir.'/tests');
	if ($shellout)
		die ($shellout."\n");

	$shellout = rrmdir($relDir.'/db/convertscripts');
	if ($shellout)
		die ($shellout."\n");

	$shellout = rrmdir($relDir.'/doc/devtools');
	if ($shellout)
		die ($shellout."\n");

	$shellout = rrmdir($relDir.'/bin');
	if ($shellout)
		die ($shellout."\n");

	removeFiles($relDir,array('.gitignore'));

	echo "Removing language file comments\n";
	foreach (scandir($relDir.'/lang') as $strip) {
		if (is_file($relDir . '/lang/' .$strip.'/language.php'))
			$shellout = shell_exec('php '.escapeshellarg(dirname(__FILE__).'/stripcomments.php').' '. escapeshellarg($relDir . '/lang/' .$strip.'/language.php').' 2>&1');
		if ($shellout)
			die ($shellout."\n");
	}
{'fraction_non_alphanumeric': 0.19453204022133636, 'fraction_numerical': 0.004045933242101505, 'mean_word_length': 3.925274725274725, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 81, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4519266', 'n_tokens_mistral': 12776, 'n_tokens_neox': 11640, 'n_words': 2862}