<commit_before><?php
use wcf\system\package\SplitNodeException;
use wcf\system\WCF;

/**
 * @author	<NAME>
 * @copyright	2001-2015 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	com.woltlab.wcf
 * @category	Community Framework
 */
$rebuildData = WCF::getSession()->getVar('__wcfUpdateRebuildComments');
if ($rebuildData === null) {
	$sql = "SELECT	COUNT(*) AS count
		FROM	wcf".WCF_N."_comment
		WHERE	responses > ?";
	$statement = WCF::getDB()->prepareStatement($sql);
	$statement->execute(array(3));
	$row = $statement->fetchSingleRow();
	
	$rebuildData = array(
		'i' => 0,
		'max' => 0
	);
	
	if ($row['count']) {
		$rebuildData['max'] = ceil($row['count'] / 50);
	}
}

if ($rebuildData['max']) {
	$offset = $rebuildData['i'] * 50;
	
	// get comments
	$sql = "SELECT		commentID
		FROM		wcf".WCF_N."_comment
		WHERE		responses > ?
		ORDER BY	commentID";
	$statement = WCF::getDB()->prepareStatement($sql, 50, $offset);
	$statement->execute(array(3));
	
	$commentData = array();
	while ($row = $statement->fetchArray()) {
		$commentData[$row['commentID']] = array();
	}
	
	if (empty($commentData)) {
		WCF::getSession()->unregister('__wcfUpdateRebuildComments');
	}
	else {
		// get responses per comment
		$sql = "SELECT		responseID
			FROM		wcf".WCF_N."_comment_response
			WHERE		commentID = ?
			ORDER BY	time";
		$statement = WCF::getDB()->prepareStatement($sql, 5);
		
		foreach ($commentData as $commentID => &$responseIDs) {<commit_msg>Processing 100 instead of 50 comments per iteration<commit_after><?php
use wcf\system\package\SplitNodeException;
use wcf\system\WCF;

/**
 * @author	<NAME>
 * @copyright	2001-2015 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	com.woltlab.wcf
 * @category	Community Framework
 */
$commentsPerRun = 100;
$rebuildData = WCF::getSession()->getVar('__wcfUpdateRebuildComments');
if ($rebuildData === null) {
	$sql = "SELECT	COUNT(*) AS count
		FROM	wcf".WCF_N."_comment
		WHERE	responses > ?";
	$statement = WCF::getDB()->prepareStatement($sql);
	$statement->execute(array(3));
	$row = $statement->fetchSingleRow();
	
	$rebuildData = array(
		'i' => 0,
		'max' => 0
	);
	
	if ($row['count']) {
		$rebuildData['max'] = ceil($row['count'] / $commentsPerRun);
	}
}

if ($rebuildData['max']) {
	$offset = $rebuildData['i'] * $commentsPerRun;
	
	// get comments
	$sql = "SELECT		commentID
		FROM		wcf".WCF_N."_comment
		WHERE		responses > ?
		ORDER BY	commentID";
	$statement = WCF::getDB()->prepareStatement($sql, $commentsPerRun, $offset);
	$statement->execute(array(3));
	
	$commentData = array();
	while ($row = $statement->fetchArray()) {
		$commentData[$row['commentID']] = array();
	}
	
	if (empty($commentData)) {
		WCF::getSession()->unregister('__wcfUpdateRebuildComments');
	}
	else {
		// get responses per comment
		$sql = "SELECT		responseID
			FROM		wcf".WCF_N."_comment_response
			WHERE		commentID = ?
			ORDER BY	time";
		$statement = WCF::getDB()->prepareStatement($sql, 5);
		
		foreach ($commentData as $commentID => &$responseIDs) {
{'fraction_non_alphanumeric': 0.1789808917197452, 'fraction_numerical': 0.012738853503184714, 'mean_word_length': 4.618962432915922, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '938484', 'n_tokens_mistral': 1219, 'n_tokens_neox': 1096, 'n_words': 225}