<commit_before><?php

/* 
 * FUPS: Forum user-post scraper. An extensible PHP framework for scraping and
 * outputting the posts of a specified user from a specified forum/board
 * running supported forum software. Can be run as either a web app or a
 * commandline script.
 *
 * Copyright (C) 2013-2014 <NAME>.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

/* File       : common.php.
 * Description: Contains defines and functions shared between FUPS scripts.
 */

require_once __DIR__.'/settings.php';

// Check before doing anything else that FUPS_CMDLINE_PHP_PATH is valid.
// This is especially important under Windows because if we supply an invalid command
// to "start" (generated by make_php_exec_cmd() below), it pops up an invisible
// error message which causes a hang until PHP timeout.
// If not Windows, assume a UNIX-like "which" command is present, otherwise use
// a custom batch file.
$cmd = (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN' ?  __DIR__.'\\would_run.bat' : 'which').' '.escapeshellarg(FUPS_CMDLINE_PHP_PATH);
exec($cmd, $dummy, $res);
if ($res !== 0) {
	exit('Fatal error: The value defined in settings.php for FUPS_CMDLINE_PHP_PATH, "'.FUPS_CMDLINE_PHP_PATH.'", does not appear to be runnable given the current working directory and your path. Exiting.');
}

define('FUPS_DONE_STR'               , 'DONE'     );
define('FUPS_FAILED_STR'             , 'EXITING'  );
define('FUPS_CANCELLED_STR'          , 'CANCELLED');
define('FUPS_MAX_TOKEN_ATTEMPTS'     ,          10);
define('FUPS_FALLBACK_FUPS_CHAIN_DURATION',   1200);

function format_html($html) {
	$flags = defined('ENT_SUBSTITUTE') ? ENT_SUBSTITUTE : (ENT_COMPAT | ENT_HTML401);
	return str_replace("\n", "<br />\n", htmlspecialchars($html, $flags));
}

function make_cancellation_filename($token) {
	return FUPS_DATADIR.$token.'.cancel.txt';
}

function make_cookie_filename($token_or_settings_filename) {
	return ($token_or_settings_filename[0] == '/' ? '' : FUPS_DATADIR).$token_or_settings_filename.'.cookies.txt';
}

function make_errs_filename($token) {
	return FUPS_DATADIR.$token.'.errs.txt';
}

function make_errs_admin_filename($token) {
	return FUPS_DATADIR.$token.'.errs.admin.txt';
}

function make_output_filename($token, $for_web = false) {
	return ($for_web ? FUPS_OUTPUTDIR_WEB : FUPS_OUTPUTDIR).$token.'.html';
}

function make_php_exec_cmd($params) {
	$args = '';
	$prefix = '';
	$redirect = '1>/dev/null';
	$bg_token = '&';
	if (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN') {
		$prefix = 'start /b ';
		$redirect = '1>NUL';
		$bg_token = '';
	}
	if (isset($params['token'])) {
		if ($args) $args .= ' ';
		$args .= '-t '.escapeshellarg($params['token']);
		$errs_fname = make_errs_filename($params['token']);
		$redirect = ' 1>>'.$errs_fname.' 2>&1';
	}
	if (isset($params['settings_filename'])) {
		if ($args) $args .= ' ';
		$args .= '-i '.escapeshellarg($params['settings_filename']);
	}
	if (isset($params['output_filename'])) {
		if ($args) $args .= ' ';
		$args .= '-o '.escapeshellarg($params['output_filename']);
	}
	if (isset($params['chained']) && $params['chained'] == true) {
		if ($args) $args .= ' ';
		$args .= '-c';
	}

	// Early return possible
	return $prefix.FUPS_CMDLINE_PHP_PATH.' -d max_execution_time=0 fups.php '.$args.' '.$redirect.' '.$bg_token;
}

function try_run_bg_proc($cmd) {
	$res = popen($cmd, 'r');
	$ret = $res !== false;
	pclose($res);
	return $ret;
}

function make_serialize_filename($token_or_settings_filename) {
	return ($token_or_settings_filename[0] == '/' ? '' : FUPS_DATADIR).$token_or_settings_filename.'.serialize.txt';
}

function make_settings_filename($token) {
	return FUPS_DATADIR.$token.'.settings.txt';
}

function make_status_filename($token) {
	return FUPS_DATADIR.$token.'.status.txt';
}

function validate_token($token, &$err) {
	$err = '';
	if (strlen($token) <> 32) {
		$err = 'A fatal error occurred: token is malformed (length).';
	} else {
		$malformed_char = false;
		for ($i = 0; $i < strlen($token); $i++) {
			$ch = $token[$i];
			if (!($ch >= '0' && $ch <= '9') && !($ch >= 'a' && $ch <= 'z')) {<commit_msg>Re-fixed filename generation for cmdline runs.<commit_after><?php

/* 
 * FUPS: Forum user-post scraper. An extensible PHP framework for scraping and
 * outputting the posts of a specified user from a specified forum/board
 * running supported forum software. Can be run as either a web app or a
 * commandline script.
 *
 * Copyright (C) 2013-2014 <NAME>.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

/* File       : common.php.
 * Description: Contains defines and functions shared between FUPS scripts.
 */

require_once __DIR__.'/settings.php';

// Check before doing anything else that FUPS_CMDLINE_PHP_PATH is valid.
// This is especially important under Windows because if we supply an invalid command
// to "start" (generated by make_php_exec_cmd() below), it pops up an invisible
// error message which causes a hang until PHP timeout.
// If not Windows, assume a UNIX-like "which" command is present, otherwise use
// a custom batch file.
$cmd = (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN' ?  __DIR__.'\\would_run.bat' : 'which').' '.escapeshellarg(FUPS_CMDLINE_PHP_PATH);
exec($cmd, $dummy, $res);
if ($res !== 0) {
	exit('Fatal error: The value defined in settings.php for FUPS_CMDLINE_PHP_PATH, "'.FUPS_CMDLINE_PHP_PATH.'", does not appear to be runnable given the current working directory and your path. Exiting.');
}

define('FUPS_DONE_STR'               , 'DONE'     );
define('FUPS_FAILED_STR'             , 'EXITING'  );
define('FUPS_CANCELLED_STR'          , 'CANCELLED');
define('FUPS_MAX_TOKEN_ATTEMPTS'     ,          10);
define('FUPS_FALLBACK_FUPS_CHAIN_DURATION',   1200);

function format_html($html) {
	$flags = defined('ENT_SUBSTITUTE') ? ENT_SUBSTITUTE : (ENT_COMPAT | ENT_HTML401);
	return str_replace("\n", "<br />\n", htmlspecialchars($html, $flags));
}

function make_cancellation_filename($token) {
	return FUPS_DATADIR.$token.'.cancel.txt';
}

function make_cookie_filename($token_or_settings_filename) {
	return FUPS_DATADIR.sanitise_filename($token_or_settings_filename).'.cookies.txt';
}

function make_errs_filename($token) {
	return FUPS_DATADIR.$token.'.errs.txt';
}

function make_errs_admin_filename($token) {
	return FUPS_DATADIR.$token.'.errs.admin.txt';
}

function make_output_filename($token, $for_web = false) {
	return ($for_web ? FUPS_OUTPUTDIR_WEB : FUPS_OUTPUTDIR).$token.'.html';
}

function make_php_exec_cmd($params) {
	$args = '';
	$prefix = '';
	$redirect = '1>/dev/null';
	$bg_token = '&';
	if (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN') {
		$prefix = 'start /b ';
		$redirect = '1>NUL';
		$bg_token = '';
	}
	if (isset($params['token'])) {
		if ($args) $args .= ' ';
		$args .= '-t '.escapeshellarg($params['token']);
		$errs_fname = make_errs_filename($params['token']);
		$redirect = ' 1>>'.$errs_fname.' 2>&1';
	}
	if (isset($params['settings_filename'])) {
		if ($args) $args .= ' ';
		$args .= '-i '.escapeshellarg($params['settings_filename']);
	}
	if (isset($params['output_filename'])) {
		if ($args) $args .= ' ';
		$args .= '-o '.escapeshellarg($params['output_filename']);
	}
	if (isset($params['chained']) && $params['chained'] == true) {
		if ($args) $args .= ' ';
		$args .= '-c';
	}

	// Early return possible
	return $prefix.FUPS_CMDLINE_PHP_PATH.' -d max_execution_time=0 fups.php '.$args.' '.$redirect.' '.$bg_token;
}

function try_run_bg_proc($cmd) {
	$res = popen($cmd, 'r');
	$ret = $res !== false;
	pclose($res);
	return $ret;
}

function sanitise_filename($filename) {
	$sanitised = preg_replace('/[^A-Za-z0-9_\-\.]/', '_', $filename);
	return $sanitised !== null ? $sanitised : $filename;
}

function make_serialize_filename($token_or_settings_filename) {
	return FUPS_DATADIR.sanitise_filename($token_or_settings_filename).'.serialize.txt';
}

function make_settings_filename($token) {
	return FUPS_DATADIR.$token.'.settings.txt';
}

function make_status_filename($token) {
	return FUPS_DATADIR.$token.'.status.txt';
}

function validate_token($token, &$err) {
	$err = '';
	if (strlen($token) <> 32) {
		$err = 'A fatal error occurred: token is malformed (length).';
	} else {
		$malformed_char = false;
		for ($i = 0; $i < strlen($token); $i++) {
			$ch = $token[$i];
			if (!($ch >= '0' && $ch <= '9') && !($ch >= 'a' && $ch <= 'z')) {
{'fraction_non_alphanumeric': 0.1585378590078329, 'fraction_numerical': 0.00751958224543081, 'mean_word_length': 4.340769659788065, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '440708', 'n_tokens_mistral': 3389, 'n_tokens_neox': 3263, 'n_words': 873}